{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-tuning Pre-trained Model for Perturbation Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import copy\n",
    "from pathlib import Path\n",
    "from typing import Iterable, List, Tuple, Dict, Union, Optional\n",
    "import warnings\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from torchtext.vocab import Vocab\n",
    "from torchtext._torchtext import (\n",
    "    Vocab as VocabPybind,\n",
    ")\n",
    "from torch_geometric.loader import DataLoader\n",
    "from gears import PertData, GEARS\n",
    "from gears.inference import compute_metrics, deeper_analysis, non_dropout_analysis\n",
    "from gears.utils import create_cell_graph_dataset_for_prediction\n",
    "\n",
    "sys.path.insert(0, \"../\")\n",
    "\n",
    "import scgpt as scg\n",
    "from scgpt.model import TransformerGenerator\n",
    "from scgpt.loss import (\n",
    "    masked_mse_loss,\n",
    "    criterion_neg_log_bernoulli,\n",
    "    masked_relative_error,\n",
    ")\n",
    "from scgpt.tokenizer import tokenize_batch, pad_batch, tokenize_and_pad_batch\n",
    "from scgpt.tokenizer.gene_tokenizer import GeneVocab\n",
    "from scgpt.utils import set_seed, map_raw_id_to_vocab_id, compute_perturbation_metrics\n",
    "\n",
    "matplotlib.rcParams[\"savefig.transparent\"] = False\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "set_seed(42)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Training Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# settings for data prcocessing\n",
    "pad_token = \"<pad>\"\n",
    "special_tokens = [pad_token, \"<cls>\", \"<eoc>\"]\n",
    "pad_value = 0  # for padding values\n",
    "pert_pad_id = 0\n",
    "include_zero_gene = \"all\"\n",
    "max_seq_len = 1536\n",
    "\n",
    "# settings for training\n",
    "MLM = True  # whether to use masked language modeling, currently it is always on.\n",
    "CLS = False  # celltype classification objective\n",
    "CCE = False  # Contrastive cell embedding objective\n",
    "MVC = False  # Masked value prediction for cell embedding\n",
    "ECS = False  # Elastic cell similarity objective\n",
    "amp = True\n",
    "load_model = \"../save/scGPT_human\"\n",
    "load_param_prefixs = [\n",
    "    \"encoder\",\n",
    "    \"value_encoder\",\n",
    "    \"transformer_encoder\",\n",
    "]\n",
    "\n",
    "# settings for optimizer\n",
    "lr = 1e-4  # or 1e-4\n",
    "batch_size = 64\n",
    "eval_batch_size = 64\n",
    "epochs = 15\n",
    "schedule_interval = 1\n",
    "early_stop = 10\n",
    "\n",
    "# settings for the model\n",
    "embsize = 512  # embedding dimension\n",
    "d_hid = 512  # dimension of the feedforward network model in nn.TransformerEncoder\n",
    "nlayers = 12  # number of nn.TransformerEncoderLayer in nn.TransformerEncoder\n",
    "nhead = 8  # number of heads in nn.MultiheadAttention\n",
    "n_layers_cls = 3\n",
    "dropout = 0  # dropout probability\n",
    "use_fast_transformer = True  # whether to use fast transformer\n",
    "\n",
    "# logging\n",
    "log_interval = 100\n",
    "\n",
    "# dataset and evaluation choices\n",
    "data_name = \"adamson\"\n",
    "split = \"simulation\"\n",
    "if data_name == \"norman\":\n",
    "    perts_to_plot = [\"SAMD1+ZBTB1\"]\n",
    "elif data_name == \"adamson\":\n",
    "    perts_to_plot = [\"KCTD16+ctrl\"]\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving to save/dev_perturb_adamson-Jun03-12-45\n",
      "scGPT - INFO - Running on 2024-06-03 12:45:07\n"
     ]
    }
   ],
   "source": [
    "save_dir = Path(f\"./save/dev_perturb_{data_name}-{time.strftime('%b%d-%H-%M')}/\")\n",
    "save_dir.mkdir(parents=True, exist_ok=True)\n",
    "print(f\"saving to {save_dir}\")\n",
    "\n",
    "logger = scg.logger\n",
    "scg.utils.add_file_handler(logger, save_dir / \"run.log\")\n",
    "# log running date and current git commit\n",
    "logger.info(f\"Running on {time.strftime('%Y-%m-%d %H:%M:%S')}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found local copy...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Local copy of pyg dataset is detected. Loading...\n",
      "Done!\n",
      "Local copy of split is detected. Loading...\n",
      "Simulation split test composition:\n",
      "combo_seen0:0\n",
      "combo_seen1:0\n",
      "combo_seen2:0\n",
      "unseen_single:22\n",
      "Done!\n",
      "Creating dataloaders....\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "pert_data = PertData(\"./data\")\n",
    "pert_data.load(data_name=data_name)\n",
    "pert_data.prepare_split(split=split, seed=1)\n",
    "pert_data.get_dataloader(batch_size=batch_size, test_batch_size=eval_batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scGPT - INFO - match 4399/5060 genes in vocabulary of size 60697.\n",
      "scGPT - INFO - Resume model from ../save/scGPT_human/best_model.pt, the model args will override the config ../save/scGPT_human/args.json.\n"
     ]
    }
   ],
   "source": [
    "if load_model is not None:\n",
    "    model_dir = Path(load_model)\n",
    "    model_config_file = model_dir / \"args.json\"\n",
    "    model_file = model_dir / \"best_model.pt\"\n",
    "    vocab_file = model_dir / \"vocab.json\"\n",
    "\n",
    "    vocab = GeneVocab.from_file(vocab_file)\n",
    "    for s in special_tokens:\n",
    "        if s not in vocab:\n",
    "            vocab.append_token(s)\n",
    "\n",
    "    pert_data.adata.var[\"id_in_vocab\"] = [\n",
    "        1 if gene in vocab else -1 for gene in pert_data.adata.var[\"gene_name\"]\n",
    "    ]\n",
    "    gene_ids_in_vocab = np.array(pert_data.adata.var[\"id_in_vocab\"])\n",
    "    logger.info(\n",
    "        f\"match {np.sum(gene_ids_in_vocab >= 0)}/{len(gene_ids_in_vocab)} genes \"\n",
    "        f\"in vocabulary of size {len(vocab)}.\"\n",
    "    )\n",
    "    genes = pert_data.adata.var[\"gene_name\"].tolist()\n",
    "\n",
    "    # model\n",
    "    with open(model_config_file, \"r\") as f:\n",
    "        model_configs = json.load(f)\n",
    "    logger.info(\n",
    "        f\"Resume model from {model_file}, the model args will override the \"\n",
    "        f\"config {model_config_file}.\"\n",
    "    )\n",
    "    embsize = model_configs[\"embsize\"]\n",
    "    nhead = model_configs[\"nheads\"]\n",
    "    d_hid = model_configs[\"d_hid\"]\n",
    "    nlayers = model_configs[\"nlayers\"]\n",
    "    n_layers_cls = model_configs[\"n_layers_cls\"]\n",
    "else:\n",
    "    genes = pert_data.adata.var[\"gene_name\"].tolist()\n",
    "    vocab = Vocab(\n",
    "        VocabPybind(genes + special_tokens, None)\n",
    "    )  # bidirectional lookup [gene <-> int]\n",
    "vocab.set_default_index(vocab[\"<pad>\"])\n",
    "gene_ids = np.array(\n",
    "    [vocab[gene] if gene in vocab else vocab[\"<pad>\"] for gene in genes], dtype=int\n",
    ")\n",
    "n_genes = len(genes)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Create and train scGpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scGPT - INFO - Loading params encoder.embedding.weight with shape torch.Size([60697, 512])\n",
      "scGPT - INFO - Loading params encoder.enc_norm.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params encoder.enc_norm.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params value_encoder.linear1.weight with shape torch.Size([512, 1])\n",
      "scGPT - INFO - Loading params value_encoder.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params value_encoder.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params value_encoder.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params value_encoder.norm.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params value_encoder.norm.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.0.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.1.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.2.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.3.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.4.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.5.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.6.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.7.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.8.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.9.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.10.norm2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.self_attn.Wqkv.weight with shape torch.Size([1536, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.self_attn.Wqkv.bias with shape torch.Size([1536])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.self_attn.out_proj.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.self_attn.out_proj.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.linear1.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.linear1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.linear2.weight with shape torch.Size([512, 512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.linear2.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.norm1.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.norm1.bias with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.norm2.weight with shape torch.Size([512])\n",
      "scGPT - INFO - Loading params transformer_encoder.layers.11.norm2.bias with shape torch.Size([512])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TransformerGenerator(\n",
       "  (encoder): GeneEncoder(\n",
       "    (embedding): Embedding(60697, 512, padding_idx=60694)\n",
       "    (enc_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (value_encoder): ContinuousValueEncoder(\n",
       "    (dropout): Dropout(p=0, inplace=False)\n",
       "    (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
       "    (activation): ReLU()\n",
       "    (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "    (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (pert_encoder): Embedding(3, 512, padding_idx=0)\n",
       "  (transformer_encoder): TransformerEncoder(\n",
       "    (layers): ModuleList(\n",
       "      (0): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (1): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (2): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (3): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (4): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (5): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (6): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (7): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (8): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (9): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (10): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "      (11): FlashTransformerEncoderLayer(\n",
       "        (self_attn): FlashMHA(\n",
       "          (Wqkv): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (inner_attn): FlashAttention()\n",
       "          (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (dropout): Dropout(p=0, inplace=False)\n",
       "        (linear2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0, inplace=False)\n",
       "        (dropout2): Dropout(p=0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (decoder): AffineExprDecoder(\n",
       "    (coeff_decoder): ExprDecoder(\n",
       "      (fc): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01)\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (3): LeakyReLU(negative_slope=0.01)\n",
       "        (4): Linear(in_features=512, out_features=1, bias=True)\n",
       "      )\n",
       "    )\n",
       "    (bias_decoder): ExprDecoder(\n",
       "      (fc): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01)\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (3): LeakyReLU(negative_slope=0.01)\n",
       "        (4): Linear(in_features=512, out_features=1, bias=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (cls_decoder): ClsDecoder(\n",
       "    (_decoder): ModuleList(\n",
       "      (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "      (1): ReLU()\n",
       "      (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (3): Linear(in_features=512, out_features=512, bias=True)\n",
       "      (4): ReLU()\n",
       "      (5): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (out_layer): Linear(in_features=512, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ntokens = len(vocab)  # size of vocabulary\n",
    "model = TransformerGenerator(\n",
    "    ntokens,\n",
    "    embsize,\n",
    "    nhead,\n",
    "    d_hid,\n",
    "    nlayers,\n",
    "    nlayers_cls=n_layers_cls,\n",
    "    n_cls=1,\n",
    "    vocab=vocab,\n",
    "    dropout=dropout,\n",
    "    pad_token=pad_token,\n",
    "    pad_value=pad_value,\n",
    "    pert_pad_id=pert_pad_id,\n",
    "    use_fast_transformer=use_fast_transformer,\n",
    ")\n",
    "if load_param_prefixs is not None and load_model is not None:\n",
    "    # only load params that start with the prefix\n",
    "    model_dict = model.state_dict()\n",
    "    pretrained_dict = torch.load(model_file)\n",
    "    pretrained_dict = {\n",
    "        k: v\n",
    "        for k, v in pretrained_dict.items()\n",
    "        if any([k.startswith(prefix) for prefix in load_param_prefixs])\n",
    "    }\n",
    "    for k, v in pretrained_dict.items():\n",
    "        logger.info(f\"Loading params {k} with shape {v.shape}\")\n",
    "    model_dict.update(pretrained_dict)\n",
    "    model.load_state_dict(model_dict)\n",
    "elif load_model is not None:\n",
    "    try:\n",
    "        model.load_state_dict(torch.load(model_file))\n",
    "        logger.info(f\"Loading all model params from {model_file}\")\n",
    "    except:\n",
    "        # only load params that are in the model and match the size\n",
    "        model_dict = model.state_dict()\n",
    "        pretrained_dict = torch.load(model_file)\n",
    "        pretrained_dict = {\n",
    "            k: v\n",
    "            for k, v in pretrained_dict.items()\n",
    "            if k in model_dict and v.shape == model_dict[k].shape\n",
    "        }\n",
    "        for k, v in pretrained_dict.items():\n",
    "            logger.info(f\"Loading params {k} with shape {v.shape}\")\n",
    "        model_dict.update(pretrained_dict)\n",
    "        model.load_state_dict(model_dict)\n",
    "model.to(device)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "criterion = masked_mse_loss\n",
    "criterion_cls = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, schedule_interval, gamma=0.9)\n",
    "scaler = torch.cuda.amp.GradScaler(enabled=amp)\n",
    "\n",
    "\n",
    "def train(model: nn.Module, train_loader: torch.utils.data.DataLoader) -> None:\n",
    "    \"\"\"\n",
    "    Train the model for one epoch.\n",
    "    \"\"\"\n",
    "    model.train()\n",
    "    total_loss, total_mse = 0.0, 0.0\n",
    "    start_time = time.time()\n",
    "\n",
    "    num_batches = len(train_loader)\n",
    "    for batch, batch_data in enumerate(train_loader):\n",
    "        batch_size = len(batch_data.y)\n",
    "        batch_data.to(device)\n",
    "        x: torch.Tensor = batch_data.x  # (batch_size * n_genes, 2)\n",
    "        ori_gene_values = x[:, 0].view(batch_size, n_genes)\n",
    "        pert_flags = x[:, 1].long().view(batch_size, n_genes)\n",
    "        target_gene_values = batch_data.y  # (batch_size, n_genes)\n",
    "\n",
    "        if include_zero_gene in [\"all\", \"batch-wise\"]:\n",
    "            if include_zero_gene == \"all\":\n",
    "                input_gene_ids = torch.arange(n_genes, device=device, dtype=torch.long)\n",
    "            else:\n",
    "                input_gene_ids = (\n",
    "                    ori_gene_values.nonzero()[:, 1].flatten().unique().sort()[0]\n",
    "                )\n",
    "            # sample input_gene_id\n",
    "            if len(input_gene_ids) > max_seq_len:\n",
    "                input_gene_ids = torch.randperm(len(input_gene_ids), device=device)[\n",
    "                    :max_seq_len\n",
    "                ]\n",
    "            input_values = ori_gene_values[:, input_gene_ids]\n",
    "            input_pert_flags = pert_flags[:, input_gene_ids]\n",
    "            target_values = target_gene_values[:, input_gene_ids]\n",
    "\n",
    "            mapped_input_gene_ids = map_raw_id_to_vocab_id(input_gene_ids, gene_ids)\n",
    "            mapped_input_gene_ids = mapped_input_gene_ids.repeat(batch_size, 1)\n",
    "\n",
    "            # src_key_padding_mask = mapped_input_gene_ids.eq(vocab[pad_token])\n",
    "            src_key_padding_mask = torch.zeros_like(\n",
    "                input_values, dtype=torch.bool, device=device\n",
    "            )\n",
    "\n",
    "        with torch.cuda.amp.autocast(enabled=amp):\n",
    "            output_dict = model(\n",
    "                mapped_input_gene_ids,\n",
    "                input_values,\n",
    "                input_pert_flags,\n",
    "                src_key_padding_mask=src_key_padding_mask,\n",
    "                CLS=CLS,\n",
    "                CCE=CCE,\n",
    "                MVC=MVC,\n",
    "                ECS=ECS,\n",
    "            )\n",
    "            output_values = output_dict[\"mlm_output\"]\n",
    "\n",
    "            masked_positions = torch.ones_like(\n",
    "                input_values, dtype=torch.bool\n",
    "            )  # Use all\n",
    "            loss = loss_mse = criterion(output_values, target_values, masked_positions)\n",
    "\n",
    "        model.zero_grad()\n",
    "        scaler.scale(loss).backward()\n",
    "        scaler.unscale_(optimizer)\n",
    "        with warnings.catch_warnings(record=True) as w:\n",
    "            warnings.filterwarnings(\"always\")\n",
    "            torch.nn.utils.clip_grad_norm_(\n",
    "                model.parameters(),\n",
    "                1.0,\n",
    "                error_if_nonfinite=False if scaler.is_enabled() else True,\n",
    "            )\n",
    "            if len(w) > 0:\n",
    "                logger.warning(\n",
    "                    f\"Found infinite gradient. This may be caused by the gradient \"\n",
    "                    f\"scaler. The current scale is {scaler.get_scale()}. This warning \"\n",
    "                    \"can be ignored if no longer occurs after autoscaling of the scaler.\"\n",
    "                )\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "\n",
    "        # torch.cuda.empty_cache()\n",
    "\n",
    "        total_loss += loss.item()\n",
    "        total_mse += loss_mse.item()\n",
    "        if batch % log_interval == 0 and batch > 0:\n",
    "            lr = scheduler.get_last_lr()[0]\n",
    "            ms_per_batch = (time.time() - start_time) * 1000 / log_interval\n",
    "            cur_loss = total_loss / log_interval\n",
    "            cur_mse = total_mse / log_interval\n",
    "            # ppl = math.exp(cur_loss)\n",
    "            logger.info(\n",
    "                f\"| epoch {epoch:3d} | {batch:3d}/{num_batches:3d} batches | \"\n",
    "                f\"lr {lr:05.4f} | ms/batch {ms_per_batch:5.2f} | \"\n",
    "                f\"loss {cur_loss:5.2f} | mse {cur_mse:5.2f} |\"\n",
    "            )\n",
    "            total_loss = 0\n",
    "            total_mse = 0\n",
    "            start_time = time.time()\n",
    "\n",
    "\n",
    "def eval_perturb(\n",
    "    loader: DataLoader, model: TransformerGenerator, device: torch.device\n",
    ") -> Dict:\n",
    "    \"\"\"\n",
    "    Run model in inference mode using a given data loader\n",
    "    \"\"\"\n",
    "\n",
    "    model.eval()\n",
    "    model.to(device)\n",
    "    pert_cat = []\n",
    "    pred = []\n",
    "    truth = []\n",
    "    pred_de = []\n",
    "    truth_de = []\n",
    "    results = {}\n",
    "    logvar = []\n",
    "\n",
    "    for itr, batch in enumerate(loader):\n",
    "        batch.to(device)\n",
    "        pert_cat.extend(batch.pert)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            p = model.pred_perturb(\n",
    "                batch,\n",
    "                include_zero_gene=include_zero_gene,\n",
    "                gene_ids=gene_ids,\n",
    "            )\n",
    "            t = batch.y\n",
    "            pred.extend(p.cpu())\n",
    "            truth.extend(t.cpu())\n",
    "\n",
    "            # Differentially expressed genes\n",
    "            for itr, de_idx in enumerate(batch.de_idx):\n",
    "                pred_de.append(p[itr, de_idx])\n",
    "                truth_de.append(t[itr, de_idx])\n",
    "\n",
    "    # all genes\n",
    "    results[\"pert_cat\"] = np.array(pert_cat)\n",
    "    pred = torch.stack(pred)\n",
    "    truth = torch.stack(truth)\n",
    "    results[\"pred\"] = pred.detach().cpu().numpy().astype(np.float)\n",
    "    results[\"truth\"] = truth.detach().cpu().numpy().astype(np.float)\n",
    "\n",
    "    pred_de = torch.stack(pred_de)\n",
    "    truth_de = torch.stack(truth_de)\n",
    "    results[\"pred_de\"] = pred_de.detach().cpu().numpy().astype(np.float)\n",
    "    results[\"truth_de\"] = truth_de.detach().cpu().numpy().astype(np.float)\n",
    "\n",
    "    return results\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scGPT - INFO - | epoch   1 | 100/849 batches | lr 0.0001 | ms/batch 328.30 | loss  0.10 | mse  0.10 |\n",
      "scGPT - INFO - | epoch   1 | 200/849 batches | lr 0.0001 | ms/batch 315.96 | loss  0.09 | mse  0.09 |\n",
      "scGPT - INFO - | epoch   1 | 300/849 batches | lr 0.0001 | ms/batch 314.79 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   1 | 400/849 batches | lr 0.0001 | ms/batch 314.88 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   1 | 500/849 batches | lr 0.0001 | ms/batch 315.20 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   1 | 600/849 batches | lr 0.0001 | ms/batch 314.72 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   1 | 700/849 batches | lr 0.0001 | ms/batch 313.97 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   1 | 800/849 batches | lr 0.0001 | ms/batch 316.13 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 1: \n",
      "scGPT - INFO - {'pearson': 0.9892257294575539, 'pearson_de': 0.9536847512476039, 'pearson_delta': 0.6666769004595279, 'pearson_de_delta': 0.8330308997151171}\n",
      "scGPT - INFO - | end of epoch   1 | time: 293.53s | \n",
      "scGPT - INFO - Best model with score 0.9892\n",
      "scGPT - INFO - | epoch   2 | 100/849 batches | lr 0.0001 | ms/batch 318.85 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   2 | 200/849 batches | lr 0.0001 | ms/batch 314.84 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   2 | 300/849 batches | lr 0.0001 | ms/batch 315.07 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   2 | 400/849 batches | lr 0.0001 | ms/batch 315.34 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   2 | 500/849 batches | lr 0.0001 | ms/batch 315.28 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   2 | 600/849 batches | lr 0.0001 | ms/batch 315.23 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   2 | 700/849 batches | lr 0.0001 | ms/batch 315.95 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   2 | 800/849 batches | lr 0.0001 | ms/batch 315.18 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 2: \n",
      "scGPT - INFO - {'pearson': 0.9885074603800036, 'pearson_de': 0.9520515428485282, 'pearson_delta': 0.6710248258328121, 'pearson_de_delta': 0.8388810980745884}\n",
      "scGPT - INFO - | end of epoch   2 | time: 292.53s | \n",
      "scGPT - INFO - | epoch   3 | 100/849 batches | lr 0.0001 | ms/batch 317.79 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   3 | 200/849 batches | lr 0.0001 | ms/batch 314.78 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   3 | 300/849 batches | lr 0.0001 | ms/batch 315.49 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   3 | 400/849 batches | lr 0.0001 | ms/batch 315.06 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   3 | 500/849 batches | lr 0.0001 | ms/batch 315.57 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   3 | 600/849 batches | lr 0.0001 | ms/batch 315.81 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   3 | 700/849 batches | lr 0.0001 | ms/batch 314.85 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   3 | 800/849 batches | lr 0.0001 | ms/batch 314.68 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 3: \n",
      "scGPT - INFO - {'pearson': 0.9892629962551931, 'pearson_de': 0.9487707137368578, 'pearson_delta': 0.6665948421192589, 'pearson_de_delta': 0.8166225675462415}\n",
      "scGPT - INFO - | end of epoch   3 | time: 292.43s | \n",
      "scGPT - INFO - Best model with score 0.9893\n",
      "scGPT - INFO - | epoch   4 | 100/849 batches | lr 0.0001 | ms/batch 316.41 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   4 | 200/849 batches | lr 0.0001 | ms/batch 314.60 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   4 | 300/849 batches | lr 0.0001 | ms/batch 314.60 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   4 | 400/849 batches | lr 0.0001 | ms/batch 314.01 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   4 | 500/849 batches | lr 0.0001 | ms/batch 314.04 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   4 | 600/849 batches | lr 0.0001 | ms/batch 313.93 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   4 | 700/849 batches | lr 0.0001 | ms/batch 314.04 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   4 | 800/849 batches | lr 0.0001 | ms/batch 313.86 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 4: \n",
      "scGPT - INFO - {'pearson': 0.9883759825701637, 'pearson_de': 0.9614302664487033, 'pearson_delta': 0.6379006685656928, 'pearson_de_delta': 0.8396512058125575}\n",
      "scGPT - INFO - | end of epoch   4 | time: 291.50s | \n",
      "scGPT - INFO - | epoch   5 | 100/849 batches | lr 0.0001 | ms/batch 317.14 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   5 | 200/849 batches | lr 0.0001 | ms/batch 314.59 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   5 | 300/849 batches | lr 0.0001 | ms/batch 313.70 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   5 | 400/849 batches | lr 0.0001 | ms/batch 314.54 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   5 | 500/849 batches | lr 0.0001 | ms/batch 314.15 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   5 | 600/849 batches | lr 0.0001 | ms/batch 314.47 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   5 | 700/849 batches | lr 0.0001 | ms/batch 314.41 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   5 | 800/849 batches | lr 0.0001 | ms/batch 314.27 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 5: \n",
      "scGPT - INFO - {'pearson': 0.9898445259378498, 'pearson_de': 0.9544624666565514, 'pearson_delta': 0.6800732289350021, 'pearson_de_delta': 0.8533562199091836}\n",
      "scGPT - INFO - | end of epoch   5 | time: 291.61s | \n",
      "scGPT - INFO - Best model with score 0.9898\n",
      "scGPT - INFO - | epoch   6 | 100/849 batches | lr 0.0001 | ms/batch 317.50 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   6 | 200/849 batches | lr 0.0001 | ms/batch 314.75 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   6 | 300/849 batches | lr 0.0001 | ms/batch 315.72 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   6 | 400/849 batches | lr 0.0001 | ms/batch 314.68 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   6 | 500/849 batches | lr 0.0001 | ms/batch 315.27 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   6 | 600/849 batches | lr 0.0001 | ms/batch 315.57 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   6 | 700/849 batches | lr 0.0001 | ms/batch 315.00 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   6 | 800/849 batches | lr 0.0001 | ms/batch 315.90 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 6: \n",
      "scGPT - INFO - {'pearson': 0.9903230508676109, 'pearson_de': 0.946848227544529, 'pearson_delta': 0.6719841295966481, 'pearson_de_delta': 0.8212238190538768}\n",
      "scGPT - INFO - | end of epoch   6 | time: 292.30s | \n",
      "scGPT - INFO - Best model with score 0.9903\n",
      "scGPT - INFO - | epoch   7 | 100/849 batches | lr 0.0001 | ms/batch 318.28 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   7 | 200/849 batches | lr 0.0001 | ms/batch 315.33 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   7 | 300/849 batches | lr 0.0001 | ms/batch 315.81 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   7 | 400/849 batches | lr 0.0001 | ms/batch 314.73 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   7 | 500/849 batches | lr 0.0001 | ms/batch 314.40 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   7 | 600/849 batches | lr 0.0001 | ms/batch 315.21 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   7 | 700/849 batches | lr 0.0001 | ms/batch 314.44 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   7 | 800/849 batches | lr 0.0001 | ms/batch 314.99 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 7: \n",
      "scGPT - INFO - {'pearson': 0.9904062232661642, 'pearson_de': 0.9573263921820816, 'pearson_delta': 0.6969510807994438, 'pearson_de_delta': 0.8658731821343123}\n",
      "scGPT - INFO - | end of epoch   7 | time: 292.21s | \n",
      "scGPT - INFO - Best model with score 0.9904\n",
      "scGPT - INFO - | epoch   8 | 100/849 batches | lr 0.0000 | ms/batch 318.10 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   8 | 200/849 batches | lr 0.0000 | ms/batch 314.11 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   8 | 300/849 batches | lr 0.0000 | ms/batch 315.36 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   8 | 400/849 batches | lr 0.0000 | ms/batch 315.45 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   8 | 500/849 batches | lr 0.0000 | ms/batch 314.32 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   8 | 600/849 batches | lr 0.0000 | ms/batch 314.85 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   8 | 700/849 batches | lr 0.0000 | ms/batch 314.50 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   8 | 800/849 batches | lr 0.0000 | ms/batch 314.87 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 8: \n",
      "scGPT - INFO - {'pearson': 0.9903405889093087, 'pearson_de': 0.9582105391047795, 'pearson_delta': 0.7029724506153079, 'pearson_de_delta': 0.8825178557724992}\n",
      "scGPT - INFO - | end of epoch   8 | time: 292.06s | \n",
      "scGPT - INFO - | epoch   9 | 100/849 batches | lr 0.0000 | ms/batch 317.55 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   9 | 200/849 batches | lr 0.0000 | ms/batch 314.71 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   9 | 300/849 batches | lr 0.0000 | ms/batch 315.80 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   9 | 400/849 batches | lr 0.0000 | ms/batch 314.30 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   9 | 500/849 batches | lr 0.0000 | ms/batch 314.71 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   9 | 600/849 batches | lr 0.0000 | ms/batch 313.94 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   9 | 700/849 batches | lr 0.0000 | ms/batch 315.55 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch   9 | 800/849 batches | lr 0.0000 | ms/batch 315.73 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 9: \n",
      "scGPT - INFO - {'pearson': 0.9898455688560865, 'pearson_de': 0.9516848526507314, 'pearson_delta': 0.6928932881856177, 'pearson_de_delta': 0.8269343878951014}\n",
      "scGPT - INFO - | end of epoch   9 | time: 292.14s | \n",
      "scGPT - INFO - | epoch  10 | 100/849 batches | lr 0.0000 | ms/batch 317.94 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  10 | 200/849 batches | lr 0.0000 | ms/batch 314.88 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  10 | 300/849 batches | lr 0.0000 | ms/batch 314.65 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  10 | 400/849 batches | lr 0.0000 | ms/batch 315.04 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  10 | 500/849 batches | lr 0.0000 | ms/batch 313.73 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  10 | 600/849 batches | lr 0.0000 | ms/batch 315.28 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  10 | 700/849 batches | lr 0.0000 | ms/batch 315.31 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  10 | 800/849 batches | lr 0.0000 | ms/batch 314.58 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 10: \n",
      "scGPT - INFO - {'pearson': 0.9901226958730327, 'pearson_de': 0.9578416366552508, 'pearson_delta': 0.6865686571929335, 'pearson_de_delta': 0.8495341738808564}\n",
      "scGPT - INFO - | end of epoch  10 | time: 291.98s | \n",
      "scGPT - INFO - | epoch  11 | 100/849 batches | lr 0.0000 | ms/batch 318.32 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  11 | 200/849 batches | lr 0.0000 | ms/batch 315.35 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  11 | 300/849 batches | lr 0.0000 | ms/batch 315.20 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  11 | 400/849 batches | lr 0.0000 | ms/batch 315.13 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  11 | 500/849 batches | lr 0.0000 | ms/batch 315.23 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  11 | 600/849 batches | lr 0.0000 | ms/batch 315.90 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  11 | 700/849 batches | lr 0.0000 | ms/batch 315.02 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  11 | 800/849 batches | lr 0.0000 | ms/batch 315.15 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 11: \n",
      "scGPT - INFO - {'pearson': 0.9894306105186443, 'pearson_de': 0.9486990088146493, 'pearson_delta': 0.681914471521785, 'pearson_de_delta': 0.8097841618570546}\n",
      "scGPT - INFO - | end of epoch  11 | time: 292.41s | \n",
      "scGPT - INFO - | epoch  12 | 100/849 batches | lr 0.0000 | ms/batch 318.66 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  12 | 200/849 batches | lr 0.0000 | ms/batch 315.08 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  12 | 300/849 batches | lr 0.0000 | ms/batch 315.56 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  12 | 400/849 batches | lr 0.0000 | ms/batch 314.79 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  12 | 500/849 batches | lr 0.0000 | ms/batch 315.06 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  12 | 600/849 batches | lr 0.0000 | ms/batch 314.74 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  12 | 700/849 batches | lr 0.0000 | ms/batch 315.30 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  12 | 800/849 batches | lr 0.0000 | ms/batch 315.05 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 12: \n",
      "scGPT - INFO - {'pearson': 0.9904401900223683, 'pearson_de': 0.9556375904590835, 'pearson_delta': 0.6961162737147223, 'pearson_de_delta': 0.8505992941836349}\n",
      "scGPT - INFO - | end of epoch  12 | time: 292.21s | \n",
      "scGPT - INFO - Best model with score 0.9904\n",
      "scGPT - INFO - | epoch  13 | 100/849 batches | lr 0.0000 | ms/batch 317.68 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  13 | 200/849 batches | lr 0.0000 | ms/batch 315.37 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  13 | 300/849 batches | lr 0.0000 | ms/batch 313.83 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  13 | 400/849 batches | lr 0.0000 | ms/batch 315.42 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  13 | 500/849 batches | lr 0.0000 | ms/batch 314.69 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  13 | 600/849 batches | lr 0.0000 | ms/batch 315.00 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  13 | 700/849 batches | lr 0.0000 | ms/batch 314.41 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  13 | 800/849 batches | lr 0.0000 | ms/batch 314.50 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 13: \n",
      "scGPT - INFO - {'pearson': 0.9899041085502839, 'pearson_de': 0.9523392789334268, 'pearson_delta': 0.7005061467213647, 'pearson_de_delta': 0.8224662306348456}\n",
      "scGPT - INFO - | end of epoch  13 | time: 291.96s | \n",
      "scGPT - INFO - | epoch  14 | 100/849 batches | lr 0.0000 | ms/batch 319.04 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  14 | 200/849 batches | lr 0.0000 | ms/batch 314.66 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  14 | 300/849 batches | lr 0.0000 | ms/batch 315.45 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  14 | 400/849 batches | lr 0.0000 | ms/batch 315.56 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  14 | 500/849 batches | lr 0.0000 | ms/batch 315.45 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  14 | 600/849 batches | lr 0.0000 | ms/batch 315.86 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  14 | 700/849 batches | lr 0.0000 | ms/batch 315.05 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  14 | 800/849 batches | lr 0.0000 | ms/batch 315.16 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 14: \n",
      "scGPT - INFO - {'pearson': 0.9907336896814499, 'pearson_de': 0.9544412007321013, 'pearson_delta': 0.7098260097842066, 'pearson_de_delta': 0.8567521357575016}\n",
      "scGPT - INFO - | end of epoch  14 | time: 292.49s | \n",
      "scGPT - INFO - Best model with score 0.9907\n",
      "scGPT - INFO - | epoch  15 | 100/849 batches | lr 0.0000 | ms/batch 319.35 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  15 | 200/849 batches | lr 0.0000 | ms/batch 315.52 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  15 | 300/849 batches | lr 0.0000 | ms/batch 314.74 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  15 | 400/849 batches | lr 0.0000 | ms/batch 314.69 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  15 | 500/849 batches | lr 0.0000 | ms/batch 314.45 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  15 | 600/849 batches | lr 0.0000 | ms/batch 313.89 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  15 | 700/849 batches | lr 0.0000 | ms/batch 315.58 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - | epoch  15 | 800/849 batches | lr 0.0000 | ms/batch 315.39 | loss  0.08 | mse  0.08 |\n",
      "scGPT - INFO - val_metrics at epoch 15: \n",
      "scGPT - INFO - {'pearson': 0.9899097634137083, 'pearson_de': 0.9554618649590941, 'pearson_delta': 0.6949415398380963, 'pearson_de_delta': 0.8297403169182545}\n",
      "scGPT - INFO - | end of epoch  15 | time: 292.19s | \n"
     ]
    }
   ],
   "source": [
    "best_val_loss = float(\"inf\")\n",
    "best_val_corr = 0\n",
    "best_model = None\n",
    "patience = 0\n",
    "\n",
    "for epoch in range(1, epochs + 1):\n",
    "    epoch_start_time = time.time()\n",
    "    train_loader = pert_data.dataloader[\"train_loader\"]\n",
    "    valid_loader = pert_data.dataloader[\"val_loader\"]\n",
    "\n",
    "    train(\n",
    "        model,\n",
    "        train_loader,\n",
    "    )\n",
    "\n",
    "    val_res = eval_perturb(valid_loader, model, device)\n",
    "    val_metrics = compute_perturbation_metrics(\n",
    "        val_res, pert_data.adata[pert_data.adata.obs[\"condition\"] == \"ctrl\"]\n",
    "    )\n",
    "    logger.info(f\"val_metrics at epoch {epoch}: \")\n",
    "    logger.info(val_metrics)\n",
    "\n",
    "    elapsed = time.time() - epoch_start_time\n",
    "    logger.info(f\"| end of epoch {epoch:3d} | time: {elapsed:5.2f}s | \")\n",
    "\n",
    "    val_score = val_metrics[\"pearson\"]\n",
    "    if val_score > best_val_corr:\n",
    "        best_val_corr = val_score\n",
    "        best_model = copy.deepcopy(model)\n",
    "        logger.info(f\"Best model with score {val_score:5.4f}\")\n",
    "        patience = 0\n",
    "    else:\n",
    "        patience += 1\n",
    "        if patience >= early_stop:\n",
    "            logger.info(f\"Early stop at epoch {epoch}\")\n",
    "            break\n",
    "\n",
    "    # torch.save(\n",
    "    #     model.state_dict(),\n",
    "    #     save_dir / f\"model_{epoch}.pt\",\n",
    "    # )\n",
    "\n",
    "    scheduler.step()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(best_model.state_dict(), save_dir / \"best_model.pt\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(\n",
    "    model: TransformerGenerator, pert_list: List[str], pool_size: Optional[int] = None\n",
    ") -> Dict:\n",
    "    \"\"\"\n",
    "    Predict the gene expression values for the given perturbations.\n",
    "\n",
    "    Args:\n",
    "        model (:class:`torch.nn.Module`): The model to use for prediction.\n",
    "        pert_list (:obj:`List[str]`): The list of perturbations to predict.\n",
    "        pool_size (:obj:`int`, optional): For each perturbation, use this number\n",
    "            of cells in the control and predict their perturbation results. Report\n",
    "            the stats of these predictions. If `None`, use all control cells.\n",
    "    \"\"\"\n",
    "    adata = pert_data.adata\n",
    "    ctrl_adata = adata[adata.obs[\"condition\"] == \"ctrl\"]\n",
    "    if pool_size is None:\n",
    "        pool_size = len(ctrl_adata.obs)\n",
    "    gene_list = pert_data.gene_names.values.tolist()\n",
    "    for pert in pert_list:\n",
    "        for i in pert:\n",
    "            if i not in gene_list:\n",
    "                raise ValueError(\n",
    "                    \"The gene is not in the perturbation graph. Please select from GEARS.gene_list!\"\n",
    "                )\n",
    "\n",
    "    model.eval()\n",
    "    device = next(model.parameters()).device\n",
    "    with torch.no_grad():\n",
    "        results_pred = {}\n",
    "        for pert in pert_list:\n",
    "            cell_graphs = create_cell_graph_dataset_for_prediction(\n",
    "                pert, ctrl_adata, gene_list, device, num_samples=pool_size\n",
    "            )\n",
    "            loader = DataLoader(cell_graphs, batch_size=eval_batch_size, shuffle=False)\n",
    "            preds = []\n",
    "            for batch_data in loader:\n",
    "                pred_gene_values = model.pred_perturb(\n",
    "                    batch_data, include_zero_gene, gene_ids=gene_ids, amp=amp\n",
    "                )\n",
    "                preds.append(pred_gene_values)\n",
    "            preds = torch.cat(preds, dim=0)\n",
    "            results_pred[\"_\".join(pert)] = np.mean(preds.detach().cpu().numpy(), axis=0)\n",
    "\n",
    "    return results_pred\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_perturbation(\n",
    "    model: nn.Module, query: str, save_file: str = None, pool_size: int = None\n",
    ") -> matplotlib.figure.Figure:\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "    import seaborn as sns\n",
    "\n",
    "    sns.set_theme(style=\"ticks\", rc={\"axes.facecolor\": (0, 0, 0, 0)}, font_scale=1.5)\n",
    "\n",
    "    adata = pert_data.adata\n",
    "    gene2idx = pert_data.node_map\n",
    "    cond2name = dict(adata.obs[[\"condition\", \"condition_name\"]].values)\n",
    "    gene_raw2id = dict(zip(adata.var.index.values, adata.var.gene_name.values))\n",
    "\n",
    "    de_idx = [\n",
    "        gene2idx[gene_raw2id[i]]\n",
    "        for i in adata.uns[\"top_non_dropout_de_20\"][cond2name[query]]\n",
    "    ]\n",
    "    genes = [\n",
    "        gene_raw2id[i] for i in adata.uns[\"top_non_dropout_de_20\"][cond2name[query]]\n",
    "    ]\n",
    "    truth = adata[adata.obs.condition == query].X.toarray()[:, de_idx]\n",
    "    if query.split(\"+\")[1] == \"ctrl\":\n",
    "        pred = predict(model, [[query.split(\"+\")[0]]], pool_size=pool_size)\n",
    "        pred = pred[query.split(\"+\")[0]][de_idx]\n",
    "    else:\n",
    "        pred = predict(model, [query.split(\"+\")], pool_size=pool_size)\n",
    "        pred = pred[\"_\".join(query.split(\"+\"))][de_idx]\n",
    "    ctrl_means = adata[adata.obs[\"condition\"] == \"ctrl\"].to_df().mean()[de_idx].values\n",
    "\n",
    "    pred = pred - ctrl_means\n",
    "    truth = truth - ctrl_means\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=[16.5, 4.5])\n",
    "    plt.title(query)\n",
    "    plt.boxplot(truth, showfliers=False, medianprops=dict(linewidth=0))\n",
    "\n",
    "    for i in range(pred.shape[0]):\n",
    "        _ = plt.scatter(i + 1, pred[i], color=\"red\")\n",
    "\n",
    "    plt.axhline(0, linestyle=\"dashed\", color=\"green\")\n",
    "\n",
    "    ax.xaxis.set_ticklabels(genes, rotation=90)\n",
    "\n",
    "    plt.ylabel(\"Change in Gene Expression over Control\", labelpad=10)\n",
    "    plt.tick_params(axis=\"x\", which=\"major\", pad=5)\n",
    "    plt.tick_params(axis=\"y\", which=\"major\", pad=5)\n",
    "    sns.despine()\n",
    "\n",
    "    if save_file:\n",
    "        fig.savefig(save_file, bbox_inches=\"tight\", transparent=False)\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABWkAAAImCAYAAAAhRAjoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAADCPUlEQVR4nOzdeXwN9/7H8ffkJCEbQSxBUERFSzWIq6UbvVVVQrXVSqubtrbqfrmtttTWnaK6q62rta2t9tpqC21JSlq0RIKQIIhs5/dHfskVss45yck55/V8PDwiM9+Z+Qw5k5n3fOc7htVqtQoAAAAAAAAA4BAeji4AAAAAAAAAANwZIS0AAAAAAAAAOBAhLQAAAAAAAAA4ECEtAAAAAAAAADgQIS0AAAAAAAAAOBAhLQAAAAAAAAA4ECEtAAAAAAAAADgQIS0AAAAAAAAAOBAhLQAAAAAAAAA4ECEtAAAAAAAAADgQIS0AAAAAAAAAOBAhLQAAAAAAAAA4kGdJGm3bts1uG2zXrp3d1gUAAAAAAAAAzs6wWq3W4ho1b95chmHYvjHDUExMjM3rAQAAAAAAAABXUaKetJJUgiy3XNYBAAAAAAAAAK6kRD1pAQAAAAAAAABlgxeHAQAAAAAAAIADEdICAAAAAAAAgAOVeEzakoiPj9eJEyckSTVq1FC9evXsuXoAAAAAAAAAcDk2h7THjh3Txx9/rMWLFyslJSXfvMDAQHXv3l0DBgxQrVq1bN0UAAAAAAAAALgcm14ctmPHDg0ePFinTp1SYasxDEOBgYGaOnWqwsPDTRcKAAAAAAAAAK7IdEh74sQJdevWTadOnZK/v7/69u2r66+/XrVr15YkHT16VJs2bdI333yj06dPq2rVqlqyZIlq1Khh1x0AAAAAAAAAAGdmOqR966239Nlnn6lx48aaPn16Xjh7qaNHj+rhhx/WgQMH9Oijj+r555+3qWAAAAAAAAAAcCUeZhdct26dDMPQ66+/XmhAK0m1a9fW66+/LqvVqrVr15rdHAAAAAAAAAC4JNMhbXx8vHx8fNSmTZti27Zp00Y+Pj6Kj483uzkAAAAAAAAAcEmmQ1oAAAAAAAAAgO1Mh7T16tXT+fPntWvXrmLb7ty5U+fPn1e9evXMbg4AAAAAAAAAXJLpkLZTp06yWq0aOXKkTp48WWi7EydO6JVXXpFhGLrhhhvMbg4AAAAAAAAAXJJhtVqtZhZMSkpSt27ddObMGVWpUkX33XefOnTokPcSscTERG3evFnffPONUlJSVKVKFS1ZskRBQUF23QEAAAAAAAAAcGamQ1pJ2rp1q4YMGaLTp0/LMIwC21itVlWpUkVTp05Vu3btTBcKAAAAAAAAAK7IppBWyukxO23aNC1btkynTp3KN69q1arq1q2bnnzyybwetgAAAAAAAACA/7E5pL3YoUOH8sanrV69ukJCQuy1agAAAAAAAABwSaZD2gcffFCGYej1119XgwYN7F0XAAAAAAAAALgFT7MLRkdHy9PTk4AWAAAAAAAAAGzgYXbBGjVqyMvLy561AAAAAAAAAIDbMR3Stm3bVqmpqTp48KAdywEAAAAAAAAA92I6pH300UdlsVg0YcIE2fHdYwAAAAAAAADgVkyHtC1atNC7776rrVu36r777tOKFSuUlJREYAsAAAAAAAAApWBYTaaqYWFhpd+YYSgmJsbM5gAAAAAAAADAJXmaXZAeswAAAAAAAABgO9Mh7cyZM+1ZBwAAAAAAAAC4JdPDHQAAAAAAAAAAbGf6xWEAAAAAAAAAANuZDmk7d+6se+65p8Tt77//fnXp0sXs5gAAAAAAAADAJZkekzY+Pl4XLlwocfvExEQlJCSY3RwAAAAAAAAAuKRyG+4gKytLHh6MrgAAAAAAAAAAFyuX1DQtLU0nTpyQn59feWwOAAAAAAAAAJxGiYc7OHLkiOLj4/NNy8jI0Pbt22W1Wgtcxmq16vTp0/rhhx+UmZmpZs2a2VYtAAAAAAAAALiYEoe08+fP19SpU/NNO336tB544IFil7VarTIMQ/fee2/pKwQAAAAAAAAAF1aqF4dd3GPWMIxCe9Be3Mbf31+hoaHq27ev7rzzTnNVAgAAAAAAAICLMqzFJa2FaN68uYKCgrRhwwZ71wQAAAAAAAAAbqNUPWkvFhkZqYCAAHvWAgAAAAAAAABux3RPWgAAAAAAAACA7TwcXQAAAAAAAAAAuDPTwx3kSk1N1dq1a7V3716dOnVKGRkZhbY1DEPjxo2zdZMAAAAAAAAA4DJsGu5g/vz5Gjt2rM6dO5c3raDVGYYhq9UqwzAUGxtrdnMAAAAAAAAA4HJM96Rdv369XnrpJVmtVlWqVEmtW7dWrVq15Olpc+dcAAAAAAAAAHAbphPVTz/9VFarVa1bt9YHH3yg6tWr27MuAAAAAAAAAHALpoc7aNu2rc6ePaulS5eqUaNGdi4LAAAAAAAAANyDh9kFs7Ky5OvrS0ALAAAAAAAAADYwHdKGhIQoPT1dWVlZ9qwHAAAAAAAAANyK6ZC2R48eyszM1M8//2zPegAAAAAAAADArZgOafv376+WLVtq1KhROnjwoB1LAgAAAADpgQce0JVXXqnJkyc7uhQAAIAy5Wl2wcWLF6tnz556//331bNnT91222265ppr5OfnV+RykZGRZjcJAACACmTy5MmaMmWKJGnv3r2Ftps3b55GjhyprKwstW3bVh9++KECAgLy5q9bt04rV65UdHS0jh8/rrNnz8rf318hISG69tpr1b17d11zzTWXbbO06tWrp9WrV0uShg8frgULFuSb7+HhIV9fXwUEBCgkJERhYWHq2LGjOnbsKA+Povs2bN++XXv27FFMTIz27Nmj/fv3KysrSxEREZo1a1aJa0xPT9d3332n5cuX66+//tKpU6cUGBio+vXrq23bturXr5+Cg4NLv/PlJDdM7dWrl+rXr+/gagAAAJyH6ZB2+PDhMgxDkmS1WvXDDz/ohx9+KHIZwzAIaQEAANzIF198oQkTJshqtermm2/WxIkTVblyZUnSgQMH9Pzzz2v37t157S0WiwICAnTmzBn9/vvv+v333zVz5ky1b99eEydOlK+vr4KCggrcVlJSkiTJ19dXvr6+l82vVq3aZdM8PDxUvXr1vO/PnTunhIQEJSQkaOvWrZoxY4aCg4M1YsQI3XbbbYXuZ79+/Ur2D1KEffv2afDgwfrnn38kSZ6envLz81NSUpKOHz+unTt3qnXr1hU6pM0N0CMiIghpAQAASsF0SFu3bl171gEAAAAXM3HiRE2bNk2SdOedd2rChAny9Mw5/fztt9/06KOP6vTp0/L19dUDDzygbt266corr5RhGMrOztb+/fu1cuVKzZo1S1u2bNHRo0f16KOP6tFHHy1we1deeaUk6ZFHHtHQoUNLVGNwcHBe79pc6enp2rt3r9atW6evvvpKCQkJeuqpp/TEE0/o2WefLXA9lStXVrNmzdSiRQtdffXVWrZsmTZs2FCiGiTp4MGDeuCBB5SSkqKIiAgNHTpUbdq0kcViUXp6ug4cOKA1a9aoVq1aJV6nJM2fP18jRoxQr169NGHChFItCwAAgPJjOqS99GQWAAAAkHKesnr99dc1Z84cSVJUVJRefvnlvKewkpOTNXToUJ0+fVq1atXS559/rtDQ0Hzr8PDwUNOmTdW0aVP1799f48ePz1u+rHl7e6tly5Zq2bKl+vXrp2HDhmnLli366KOPFBoaqjvvvPOyZaKjo2WxWPK+37FjR4m3Z7VaNXz4cKWkpKhLly56//33863L29tbV155ZV4IDQAAANdjOqQFAAAALpWZmanhw4fnDYM1ePBgPfXUU/nafPrpp0pMTJQkvfvuu5cFtJfy8fHR6NGjlZ2dXTZFF6FatWqaMmWKunfvrqNHj2rixInq2rWrvLy88rW7OFQtrQ0bNmjnzp3y8vLS6NGjbVqXvSUkJGjWrFnauHGjDh8+rIyMDNWqVUuhoaG67bbbdPvtt6tSpUqXjfH74IMP5lvPxeMBb9myJW/+3r17FRMTo88++0zbtm3TiRMnFB4eXqpxfAEAAFwBIS0AAADs4sKFCxo2bJjWrFkjwzD03//+97KwLjMzU998840kqUOHDmrXrl2J11/cy7vKSpUqVdS/f3+9+eabOnz4sLZv364OHTrYbf0LFy6UJHXs2FE1atSw23pttXDhQr3yyiu6cOGCJMnLy0t+fn5KSEjQoUOHtHr1al155ZUKCwuTv7+/goKC8sYFrlq1ar4gu6DxgCVp+fLleu6555SRkSF/f/8KFVADAACUJ7uFtHFxcdq9e7dOnDghSapRo4Zatmyppk2b2msTAAAAqKBSU1M1cOBAbd26VZ6enho7dmyBL4zdvXu3zpw5I0m69dZby7lK82666Sa9+eabkqRt27bZNaSNjo6WJF199dVKTk7WRx99pJUrVyoxMVF+fn5q0aKFIiMjdeedd5ZbUL127VoNHz5cVqtV4eHheu655xQeHi4PDw+lp6frt99+06JFi/KC2Jdfflkvv/xy3pAMkydPVvv27YvdzvDhw3XdddfpP//5j5o0aSIpZ3xeAAAAd2NzSLt+/Xq99dZbiouLK3B+s2bN9MILL6hjx462bgoAAAAV1IMPPqg9e/aoUqVKmjhxom655ZYC2118zhgWFlZe5dmscePG8vLyUkZGhv755x+7rTc9PV1HjhyRlBN033nnnTp+/Lg8PT3l5+enU6dOadOmTdq0aZOWLFmiyZMny9vb227bL0hmZqZef/11Wa1WtWnTRl988UW+bXp7e6tt27Zq27atzdtq2rSppk2blq8HbaNGjWxeLwAAgLOx6Vb87Nmz9cQTTyguLk5Wq1UeHh6qUaOGatSoIYvFIqvVqr1792rAgAF5L44AAACA69mzZ48kqU+fPoUGtJKUkpKS9/eqVauWdVl2YxhGXr2nTp2y23ovXteMGTN0+vRpjR49Wjt27NDWrVu1adMmRUVFScrp3Zrbm7csbdmyRYcPH5YkjRgxokxD4UcffZQhDgAAAGRDT9o//vhD48aNU3Z2tq655hoNHjxY//rXv/JO4tLT0/XLL7/ogw8+0K5duzRu3Di1adNGzZs3t1vxAAAAqBiuvfZa7dy5U3PmzFGjRo0uG4sWBbv4ZWjZ2dl69tlnde+99+ZNq169ukaOHKn4+HitWbNGX3/9tQYOHJhv7Nro6GgNHTq0wPWnpaVJkpYsWaL169cX2Oall15St27d8r7fuXOnJKlmzZpq2bKl+Z0rgfDw8DJdPwAAgLMw3ZN2+vTpys7O1s0336wvv/xSN9xww2WPQd1www2aM2eObr75ZmVlZWnGjBl2KRoAAAAVy6effpoXuI0dO1ZffPFFge0CAwPz/m7PHqllzWq16vTp05Ly74Ot/Pz88v7u4+Ojfv36FdjusccekyRlZGRoy5Yt+eZlZGQoKSmpwD+pqamScl7qVlib3CA31/HjxyVJdevWtdt+FqYivSgNAADAkUz3pN22bZsMw9BLL71U5CNKFotF//3vf7VmzZrLTigBAADgGvz9/fXpp59qwIAB2rFjh8aPHy+r1aqHH344X7vQ0NC8v8fGxjpNT8r9+/crPT1dktSgQQO7rdff31/+/v5KTU1VSEhI3ou4LnXxy3jj4+PzzWvfvr327t1b4HLz58/XiBEj1KtXL02YMKFENRmGUcLqbcdQBwAAADlM96RNSkpSQECA6tevX2zbkJAQValSRUlJSWY3BwAAgArOz89Pn376qdq1aydJmjBhgj777LN8ba6++moFBARIklasWFHuNZq1du3avL9HRETYdd3NmjUrto3Vas37e1mHqEFBQZKU90IzAAAAlD3TIW3lypV1/vx5ZWZmFts2MzNT58+fV+XKlc1uDgAAAE7A19dXH3/8cV6Q+eabb+qTTz7Jm+/p6al77rlHkrR582Zt27atxOu+ePzW8nT69GnNnDlTUk4v2jZt2th1/ddff70k6dChQ8rIyCiwzV9//ZX395J0krBFbu/m48eP6/fffy/VsrkB8sWhMgAAAIpnOqRt3LixMjMztXz58mLbLlu2TBkZGWrcuLHZzQEAAMBJ5Aa17du3lyS9/fbb+vjjj/PmP/bYY6pVq5Yk6dlnn1VcXFyR60tLS9Nrr72mffv2lV3RhUhJSdHQoUOVmJgoSXrmmWfk6Wl6xLAC9ezZU15eXjp//rzmzJlTYJvcoNvHx0cdOnSw6/Yv1b59e4WEhEiSxo8fnzfMQ0n4+/tLks6cOVMmtQEAALgq0yFt165dZbVaNWrUKG3evLnQdps2bdKoUaNkGIZuv/12s5sDAACAE/Hx8dHHH3+cFyi+8847+vDDDyVJ1atX1+TJk+Xv769jx47pnnvu0bvvvqt9+/bl9cC0Wq3666+/9Mknn6hLly766quvyq13ZkZGhnbv3q0pU6bojjvu0C+//CJJGjhwoLp161bgMmfPntXJkyfz/uQGmxkZGfmmF/SytJCQED3wwAOSpPfee0/ffvutLly4IEk6efKkxowZkzfcwmOPPaaqVavae5fzsVgsGjlypAzD0I4dO/TQQw9p+/bteT2Z09PTtWXLFj3//PP6888/8y2bO+bwDz/8oPPnz5dpnQAAAK7EdDeA+++/X/PmzVNcXJweeeQRtW7dWtddd51q164tSUpMTNTmzZu1a9cuWa1WhYaG6r777rNb4QAAAKjYKleurA8//FCDBg3Sxo0b9d577yk7O1uDBg1S69at9e233+qFF17Qnj179NFHH+mjjz6Sp6dn3ou0Lh5Wq2PHjqpTp47da0xISMgbbkDK6bV79uzZfIFw3bp19dJLL6lLly6Fruf111/XggULLpu+c+fOfD1f69Wrp9WrV1/W7vnnn1dCQoKWLl2qkSNHavTo0fLz89OpU6fyarnrrrs0aNAgU/tZWjfeeKMmTJigkSNHaseOHerXr5+8vb3l6+ub7//m0Ucfzbdc3759FR0dreXLl2v16tWqXr26PD09Vbt2bX311VflUjsAAIAzMh3Sent769NPP9XQoUP122+/aefOndq1a1e+NrknlNdcc43ef/99eXt721QsAAAAnEvlypU1bdo0DRo0SBs2bNCkSZOUnZ2tIUOGqEmTJpo/f77Wrl2rFStWKDo6WklJSUpNTZW/v79CQkLUpk0b9ejRQ1dddVWZ1JednZ33clvDMOTr66vatWsrJCRELVq0UKdOnXT99dfLw8P0A2glYrFYNHHiRHXt2lXfffedYmJidObMGdWoUUOtWrVS3759deONN5ZpDZeKjIxU27ZtNXPmTG3cuFFHjhzRhQsXVLduXTVr1kz//ve/1aRJk3zL9OzZU5L0zTffaN++fTp+/LjDxhIGAABwJobVxufGsrOztWzZMi1dulS7d+/WiRMnJEk1atTQ1VdfrW7duum2224r8xNbAAAAAAAAAHBGNoe0AAAAAAAAAADz6N4KAAAAAAAAAA5UqjFpDx06pH379sliseimm24qtr3VatW6deuUlZWlK6+8UvXr1zdbJwAAAAAAAAC4pFL1pB02bJiGDBmi3bt3l6i9YRjas2ePhgwZoueff95UgQAAAAAAAADgykoc0m7evFkxMTGqX7++Bg0aVOINDBw4UCEhIfr111+1fft2U0UCAAAAAAAAgKsqcUi7dOlSGYah/v37y8Oj5B1wPTw81L9/f1mtVi1evNhUkeXNarUqOjpab7/9tu677z61b99eV111lf71r3/pkUce0ffffy97vW8tKipKUVFRdlkXAAAAAAAAAOdT4rT1119/laQSjUV7qdxldu3aVeplHeGXX37Rfffdp08++UTR0dEKCAjQlVdeKavVqo0bN+qFF17Qk08+qfT0dJu3lZCQoISEBDtUDQAAAAAAAMAZlTikTUhIkKenp6mXf9WrV09eXl6Kj48v9bKOYLVaVb9+fb300kvatGmTVq5cqfnz52vLli1644035O3trbVr12rSpEmOLhUAAAAAAACAkytxSHvu3Dn5+vqa3pCvr6/OnTtnevny1KpVKy1btkwPPvigatSokW9eZGSkBg8eLEmaO3eusrOzHVEiAAAAAAAAABdR4pA2ICBAqamppsZizc7O1pkzZ+Tv71/qZR3B399fXl5ehc6/4YYbJEkpKSk6efJkeZUFAAAAAAAAwAWVOKStXbu2srOztXv37lJvZM+ePcrOzladOnVKvWxFlJaWlvf3ypUrO7ASAAAAAAAAAM7Os6QN27Ztq71792rhwoVq2bJlqTayYMECGYahtm3blrrAimjx4sWSpObNm5eod3Dnzp0LnZeQkKDg4GC71QYAAAAAAADAuZS4J223bt1ktVr17bffavv27SXewPbt2/Xtt99Kkm6//fbSV1jB7N69W19//bUk6fHHH3dwNQAAAAAAAACcnWEtxSCzjz76qDZu3Ch/f3+9+uqruvPOO4ts//3332v06NE6e/asOnTooM8//9zmgh0pKSlJd999t44cOaJbb71VU6ZMsXmdub1sV61aZfO6AAAAAAAAADifUoW0SUlJ6t27t44dOybDMNSwYUN17txZLVq0UNWqVSVJp06dUkxMjFatWqW///5bVqtVtWrV0rx581SzZs0y25GydubMGT344IOKiYnRVVddpZkzZ9rlRWiEtAAAAAAAAIB7K1VIK0mHDh3S0KFD9ccff+SswDAKbJe72mbNmmnq1KkKCQmxsVTHOXv2rB555BHt2rVLoaGhmjVrlqpVq2aXdRPSAgAAAAAAAO6txC8OyxUSEqJvv/1Wc+fO1Zdffqk///yzwHZNmzbV/fffrz59+sjb29vmQh3l/PnzeuKJJ7Rr1y41atRI06dPt1tACwAAAAAAAACl7kl7qaSkJMXFxSklJUWSFBgYqKZNmzr10Aa5Lly4oCeeeEKbN29WvXr1NGfOHAUHB9t1G/SkBQAAAAAAANxbqXvSXiooKEhBQUH2qKVCycjI0NChQ7V582bVrl1bM2bMsHtACwAAAAAAAAAeji6gIsrKytJzzz2ndevWqWbNmpoxY4ZTj6kLAAAAAAAAoOKyuSetK1q6dKmWL18uSfL29tZ///vfQtuOHDlSLVq0KK/SAAAAAAAAALgYQtoCpKen5/09Pj5e8fHxhbY9c+ZMeZQEAKbs378/b8zwgiQnJxf5MsTAwEA1bty4DCoDAAAAAAC5bH5xGGzDi8MAlJWkpCTVrl1b2dnZptdhsViUmJjokmOPAwAAAABQUdCTFgBcVFBQkOLi4grtSRsbG6uoqCjNnj1bYWFhBbYJDAwkoAUAAAAAoIwR0gKACyvJUAVhYWEKDw8vh2oAAAAAAEBBPBxdAAAAAAAAAAC4M9M9aY8cOSJJqlGjhipVqmS3ggAAAAAAAADAnZjuSXvLLbeoS5cuRb41HAAAAAAAAABQNNM9aX19feXl5aXatWvbsx4AAAAAAAAAcCume9LWq1dP58+fV1ZWlj3rAQAAAAAAAAC3Yjqk7dKlizIyMrRu3Tp71gMAAAAAAAAAbsV0SDtgwAA1aNBAr776qv744w971gQAAAAAAAAAbsP0mLQ//fST+vbtq8mTJ6tPnz7q2LGjwsPDVaNGDVkslkKXi4yMNLtJAAAAAAAAAHA5pkPa4cOHyzAMSZLVatW6deuKHfrAMAxCWgAAAAAAAAC4iOmQtm7duvasAwAAAAAAAADckumQdvXq1fasAwAAAAAAAADckukXhwEAAAAAAAAAbEdICwAAAAAAAAAOZHq4g4udPHlSW7Zs0ZEjR3T+/HkNGTLEHqsFAAAAAAAAAJdnU0ibmZmpt99+W19++aUyMjLypl8c0p46dUpdunRRWlqali5dqvr169uySQAAAAAAAABwKTYNdzBs2DDNmDFDGRkZatq0qSwWy2Vtqlatqu7duysjI0NLly61ZXMAAAAAAAAA4HJMh7SLFy/WqlWrVKNGDc2bN08//PCDAgMDC2zbtWtXSdKWLVvMbg4AAAAAAAAAXJLpkHb+/PkyDEMvvPCCWrRoUWTbVq1ayTAM/fXXX2Y3BwAAAAAAAAAuyXRIGxMTI0m67bbbim3r4+OjgIAAnThxwuzmAAAAAAAAAMAlmX5x2JkzZxQQEKDKlSuXqH12drYMwzC7OQAAgBLbv3+/UlJSCp2fnJysatWqFTo/MDBQjRs3LoPKAAAAAOBypkPaqlWr6uTJk7pw4YIqVapUZNtjx44pNTVVdevWNbs5AACAEklKSlJoaKiys7NNr8NisSgxMVFBQUF2rAwAAAAACmY6pG3RooU2bNigX375RTfeeGORbefNmydJuvbaa81uDgAAoESCgoIUFxdXaE/a2NhYRUVFafbs2QoLCyuwTWBgIAEtAAAAgHJjOqS98847tX79ek2aNElt27aVn59fge1+/vlnffDBBzIMQ5GRkWY3BwAAUGIlGaogLCxM4eHh5VANAAAAABTNppD222+/1fbt23Xvvfeqb9++ysjIkCRt3LhR8fHxWr16tX7++WdlZ2fr5ptvVqdOnexWOAAAAAAAAAC4AtMhrWEYmjp1qoYMGaJt27Zp7NixefMee+yxvL9brVZdd911evvtt22rFAAAAAAAAABckOmQVsp5ediMGTP0/fffa968efr111+Vnp6es2JPT7Vs2VL33nuvevToIQ8PD7sUDAAAAAAAAACuxKaQVpI8PDwUGRmpyMhIZWdnKyUlRdnZ2QoMDJSnp82rBwAAAAAAAACXZtcU1cPDQ9WrV7fnKgEAAAAAAADApZkeg+Dll1/W9u3b7VkLAAAAAAAAALgd0z1p586dq3nz5qlevXrq2bOnevTooYYNG9qzNgAAAAAAAABweaZ70rZp00aSdPjwYX3wwQfq2rWr+vbtq6+//lqnTp2yW4EAAAAAAAAA4MpMh7Rz5szRypUr9dRTT6lhw4ayWq3atWuXRo0apU6dOumpp57SypUrlZmZac96AQAAAAAAAMCl2PTisHr16mnQoEEaNGiQfvvtNy1atEhLlixRcnKyfvrpJ61YsUJVq1bVHXfcoZ49e6pVq1b2qhsAAAAAAAAAXILpnrSXatWqlUaOHKn169dr2rRpuu222+Tt7a2UlBR9+eWXuvfee3X77bfba3MAAAAAAAAA4BJs6klb4Ao9PXXzzTfr5ptvVmpqqpYuXao5c+bojz/+0MGDB+29OQAAAAAAAABwanbrSXup9PR0bdy4UatXr9aff/5ZVpsBAAAAAAAAAKdm9560O3bs0KJFi7Rs2TKdOXNGVqtVkhQUFKQ77rjD3psDAAAAAAAAAKdml5D277//1qJFi/T9998rPj5ekmS1WlWpUiV17txZPXv2VMeOHWWxWOyxOQAAAAAAAABwGaZD2lOnTmnx4sX6/vvv9euvv0rKCWYNw1Dbtm3Vo0cP3X777fL397dbsQAAAAAAAADgakyHtB07dlRmZmbecAYNGzZUz5491bNnT9WrV89uBQIAAAAAAACAKzMd0mZkZKhq1arq1q2bIiMjdc0119izLgAAAAAAAABwC6ZD2ilTpujGG2+Ul5eXPesBAAAAAAAAALdiOqTt0qWLPesAAAAAAAAAALdkOqS9VGpqqmJiYnTixAlJUo0aNdSiRQteHAYAAAAAAAAARbA5pN27d6/ee+89rV+/XtnZ2fnmeXh46MYbb9SwYcN05ZVX2ropAAAAAAAAAHA5HrYs/NNPP+mee+7RunXrlJWVJavVmu9PVlaW1qxZo3vuuUcrVqywV80AAAAAAAAA4DJM96Q9dOiQnn/+eaWnp6tevXp67LHHdP3116tOnTqSpMTERG3cuFGfffaZDh8+rOeff14//vijQkJC7FY8AAAAAAAAADg70z1pP/vsM6Wnp6t169b6/vvvdd9996lBgwby9vaWt7e3GjRooPvuu0/ff/+9WrdurfT0dE2fPt2etQMAAAAAAACA0zMd0m7evFmGYWjUqFHy8/MrtJ2vr69GjRolq9WqjRs3mt0cAAAAAAAAALgk0yFtYmKi/Pz8SvRCsCuvvFL+/v5KTEw0uzkAAAAAAAAAcEmmQ1pPT09lZmaWqK3ValVGRoY8PU0PgQsAAAAAAAAALsl0SNuwYUNduHBB69evL7bt+vXrdeHCBTVs2NDs5gAAAAAAAADAJZnu2nrLLbcoJiZGI0eO1GeffaYmTZoU2O7PP//UK6+8IsMw1LlzZ9OFAgAAACWxf/9+paSkFDo/OTlZ1apVK3R+YGCgGjduXAaVAQAAAAUzHdI+9NBD+u6775SYmKjIyEh17dpVHTp0UO3atSXljFm7efNmLV++XBkZGapTp4769+9vt8IBAACASyUlJSk0NFTZ2dmm12GxWJSYmKigoCA7VgYAAFB2uEnt/EyHtP7+/vr000/15JNPKj4+Xj/++KN+/PHHy9pZrVbVr19f06ZNk7+/v03FAgAAAEUJCgpSXFxcoRcpsbGxioqK0uzZsxUWFlZgm8DAQAJaAADgNLhJ7RpsepNXaGiovv/+e82ZM0fLli3T3r17lZWVJSnnP/fKK69Ut27ddN9998nPz88uBQMAAABFKUkvkLCwMIWHh5dDNQAAAGWLm9SuwaaQVpL8/Pz0+OOP6/HHH1dGRoZOnTolSapataq8vLxsLhAAAAAAAABA4bhJ7fxsDmkv5uXlReoOAAAAAAAAAKXg4egCAAAAAAAAAMCdEdICAAAAAAAAgAMR0gIAAAAAAACAAxHSAgAAAAAAAIAD2fXFYQCA8hUXF6czZ86YWjY2NjbfVzMCAgIUGhpqenkAAAAAAEBICwBOKy4uTs2aNbN5PVFRUTYtv2/fPoJaAAAAAABsQEgLAE4qtwft7NmzFRYWZmodycnJqlatmqllY2NjFRUVZbonLwAAAAAAyEFICwBOLiwsTOHh4Y4uAwAAAAAAh9i/f79SUlIKnV9cB6XAwEA1bty4DCorOUJaAAAAAAAAAE4pKSlJoaGhys7ONr0Oi8WixMREBQUF2bGy0rFLSHv06FHt27dPp06dUmZmZpFtIyMj7bFJAAAAAAAAAG4uKChIcXFxhfakzR2qr6ihAgMDAx0a0Eo2hrR79+7VmDFjtH379hK1NwyDkBYAAAAAAACA3ZRkqIKKPlSg6ZB2//796tevn86ePSur1SovLy9Vr15dFovFnvUBAAAAAAAAgEszHdJOmTJFqampqlWrlkaNGqUbbrjBpQLa48ePa+PGjdq9e7d+//13xcbG6sKFC4qIiNCsWbMcXR4AAAAAAAAAF2E6pN2yZYsMw9Abb7yhDh062LOmCmHx4sUaP368o8sAAAAAAAAA4OJMh7RnzpyRt7e32rdvb896Kgx/f39dd911atmypVq2bKmYmBh98MEHji4LAAAAAAAAgIsxHdLWrFlTJ0+elIeHhz3rqTD69OmjPn365H1/9OhRB1YDAAAAAAAAwFWZTlhvvvlmpaWlKSYmxp71AAAAAAAAAIBbMR3SDhw4UNWqVdO4ceOUnp5uz5oAAAAAAAAAwG2YHu7gwoULGj9+vF588UX16tVLjzzyiFq1aiU/P78il6tbt67ZTTqtzp07FzovISFBwcHB5VgNAAAAAAAAgIrEdEh7cfB4+vRpvfzyy8UuYxgGwyMAAAAAAAAAwEVMh7RWq7VclnEFq1atKnReUb1sAQAAAAAAALg+0yFtUcEjAAAAAAAAAKBkTIe09erVs2cdAAAAAAAAAOCWPBxdAAAAAAAAAAC4M9M9aQsSHx+vEydOSJJq1KhBb1sAAAAAAAAAKIbNIe2xY8f08ccfa/HixUpJSck3LzAwUN27d9eAAQNUq1YtWzcFAAAAAAAAAC7HpuEOduzYoR49emjOnDlKTk6W1WrN9yc5OVmzZ89Wz549FR0dba+aAQAAAAAAAMBlmO5Je+LECQ0aNEinTp2Sv7+/+vbtq+uvv161a9eWJB09elSbNm3SN998o+TkZA0cOFBLlixRjRo17FZ8WUpISFBkZGTe9+np6ZKk6OhotW/fPm/6Y489pgEDBpR3eQAAAAAAAABchOmQ9vPPP9epU6fUuHFjTZ8+PS+czdW4cWN16NBBUVFRevjhh3XgwAFNnz5dzz//vM1Fl4esrKzLhm+QpMzMzHzT09LSyq8oAAAgSYqLi9OZM2dMLRsbG5vvqxkBAQEKDQ01vTwAAAAAXMx0SLtu3ToZhqHXX3/9soD2YrVr19brr7+ufv36ae3atU4T0tavX1979+51dBkAAOAScXFxatasmc3riYqKsmn5ffv2EdQCAAAAsAvTIW18fLx8fHzUpk2bYtu2adNGPj4+io+PN7s5AAAAScrrQTt79myFhYWZWkdycrKqVatmatnY2FhFRUWZ7skLAAAAAJcyHdICAAA4UlhYmMLDwx1dBgAAAADYzMPsgvXq1dP58+e1a9euYtvu3LlT58+fV7169cxuDgAAAAAAAABckumQtlOnTrJarRo5cqROnjxZaLsTJ07olVdekWEYuuGGG8xuDgAAAAAAAABckunhDh599FHNmzdPf/75p26//Xbdd9996tChQ95LxBITE7V582Z98803SklJUZUqVfTII4/YrXAAAAAAAAAAcAWmQ9qgoCBNmTJFQ4YM0alTp/TRRx/po48+uqyd1WpVlSpVNHXqVAUFBdlULAAAAAAAAAC4GtPDHUhSRESEvv/+e917772qUqWKrFZrvj9VqlTRfffdpx9++EHt2rWzV80AAAAAAAAA4DJM96TNVadOHY0aNUqjRo3SoUOH8sanrV69ukJCQmwuEAAAAAAAAABcmc0h7cVCQkIIZgEAAAAAAACgFGwa7gAAAAAAAAAAYBtCWgAAAAAAAABwoBINdxAWFiZJaty4sRYvXpxvWmkYhqGYmJhSLwcAAAAAAAAArqpEIa3Vas339dK/AwAAAAAAAADMKVFIO3PmTElS5cqVL5sGAAAAAAAAADCvRCFtREREiaYBAAAAAAAAAEqHF4cBAAAAAAAAgAOVaUh76tQpnTlzpiw3AQAAAAAAAABOzXRIe/ToUS1cuFA///zzZfPi4uLUu3dv/etf/1JERITuv/9+HThwwKZCAQAAAAAAAMAVmQ5p582bpxEjRmjr1q35pqelpenxxx9XbGysrFarrFaroqOj9fDDDys1NdXmggEAdpCVJa1dK331Vc7XrCxHVwQAAAAAgNsq0YvDCrJ582ZJUrdu3fJNX7BggRISEhQYGKjnnntOlStX1jvvvKOjR49qzpw5euKJJ2yrGABgm/nzpWHDpMOH/zetfn1p0iSpd2/H1QUAAGy2f/9+paSkFDo/OTlZ1apVK3R+YGCgGjduXAaVAQCAopgOaePj4yXpsl/gK1askGEYevbZZ3X33XdLyvlFP2DAAK1evZqQFgAcaf58qU8fyWrNPz0+Pmf63LkEtQAqvLi4ONPvPYiNjc331YyAgACFhoaaXh4oK0lJSQoNDVV2drbpdVgsFiUmJiooKMiOlQEAgOKYDmmTk5Pl7++vypUr503Lzs7Wzp07ZRiGbrvttrzp119/vTw8PBiXFgAcKSsrpwftpQGtlDPNMKSnn5Z69pQslnIvDwBKIi4uTs2aNbN5PVFRUTYtv2/fPoJaVDhBQUGKi4srtCdtbGysoqKiNHv2bIWFhRXYJjAwkIAWACogblK7PtMhbVZW1mV3aPft26fz58+rWbNmqlq1at50Dw8PValShTFpAcCR1q/PP8TBpaxW6dChnHY33VRuZQFAaeRenBQVMhWnuMe9i5Ibcpm9SALKWkmGKggLC1N4eHg5VAMAsAduUrsH0yFtzZo1deTIER06dEghISGSpPXr10uSrr322svanzt3ToGBgWY3BwCwVUKCfdsBgAMRMgEAAHfBTWr3YDqkbd26tY4cOaKpU6dq3LhxSklJ0VdffSXDMNSpU6d8bQ8dOqT09HTVrFnT5oIBACYFB9u3HQAAAACg3HCT2rV5mF2wf//+kqRFixapbdu2uvHGG3XkyBHVr19fN13ymOymTZskSS1atDBfKQDANp06SfXr54w9WxDDkEJCctoBAAAAAIByYzqkbdWqlcaNGydfX1+dO3dOGRkZaty4sSZPnixPz/wddBcuXChJat++vU3FAgBsYLFIkybl/P3SoDb3+4kTeWkYAAAAAADlzPRwB5LUq1cv3X777dq3b5+qVKmiBg0ayMMjf+6bnp6ue++9V/fcc89lPWwBAOWsd29p7lxp2LD8LxGrXz8noO3d22GlAQAAAADgrmwKaSWpcuXKatWqVaHzvb29FRkZaetmAAD20ru31LOntH59zkvCgoNzhjigBy0AAAAAAA5hc0gLAHBCFovE0w0AAAAAAFQIpsekPX36tLZt26aYmJjL5h07dkxPPfWU2rRpo3bt2umFF17QiRMnbCoUAAAAAAAAAFyR6ZB27ty5evDBBzVv3rx80zMzM/Xoo49qxYoVOnv2rM6cOaMff/xRDz30kNLT020uGAAAAAAAAABciemQduPGjZKkO+64I9/0JUuWKC4uTpUqVdKTTz6pp59+Wv7+/vrzzz/17bff2lYtAACALbKypLVrpa++yvmaleXoigAAAADA/Ji0f//9tySpWbNm+aYvXbpUhmFo6NChevTRRyVJDRo00LPPPqvly5crKirKhnIBAABMmj9fGjZMOnz4f9Pq15cmTcp5oR4AAAAAOIjpnrTJycny9fWVv79/vunbt2+XJN15551507p06SLDMBQXF2d2cwAAAObNny/16ZM/oJWk+Pic6fPnO6YuAAAAAJANPWkvXLggLy+vfNP279+vM2fOqFGjRqpVq1bedG9vb1WpUkWpqanmKwUAADAjKyunB63Vevk8q1UyDOnpp6WePSWLpdzLAwDYz/79+5WSklLo/OTkZFWrVq3Q+YGBgWrcuHEZVAYAQNFMh7Q1atTQsWPHdPz4cdWsWVOStHnzZknStddee1n7CxcuKCAgwOzmAAAAzFm//vIetBezWqVDh3La3XRTuZUFALCvpKQkhYaGKjs72/Q6LBaLEhMTFRQUZMfKAAAonumQtmXLllq1apWmT5+uF198UefPn9fXX38twzDUoUOHfG2PHj2qtLQ0NWjQwOaCAQAASiUhwb7tAAAVUlBQkOLi4grtSRsbG6uoqCjNnj1bYWFhBbYJDAwkoAUAOITpkPbee+/VypUrNX36dK1Zs0Znz57VsWPHVKNGDf373//O1/aXX36RdPlLxgAAAMpccLB92wEAKqySDFUQFham8PDwcqgGAICSMx3SdurUSUOGDNEHH3ygAwcOSJKqVaumt99+W5UrV87X9scff5QktW/f3oZSAQAFiY6ONr1sceOyFSU2Ntb0doFy1amTVL9+zkvCChqX1jBy5nfqVP61AQAAwC4YkxrOznRIK0lDhgxR79699euvv6pKlSpq1arVZePOpqenq3Xr1rrmmmt0E+O8AYDdZGZmSpIGDBjg0DoYbxwVnsUiTZok9emTE8heHNQaRs7XiRN5aRgAAICTYkxquAKbQlpJqlu3rurWrVvofG9vbw0ePNjWzQAALhEREaEtW7bI09Pcobwk47IVJyAgQKGhoaaWBcpV797S3LnSsGH5XyJWv35OQNu7t8NKQznKysp5QVxCQs7wFp06Ec4DAOACGJMarsDmkBYA4DgRERE2r4Nx2eA2eveWevYkpHNX8+cXHNJPmkRIDwAugsfd3RtjUsPZ2SWkXbVqlTZs2KAjR44oLS1NM2bMyJt37tw5/fHHHzIMQ9dee609NgcAAGCOxSIx/JL7mT8/Z7iLS8ckjo/PmT53LkEtADg5HncH4OxsCmkTEhI0ZMgQxcTESJKsVquM3LHd/p+Xl5eee+45JSYm6uuvv9Y111xjyyYBAACAksvKyulBW9BL46zWnHGJn346p5c1vaoBwGnxuDsAZ2c6pD137pweeeQRHThwQHXq1FGXLl00b948paWl5Wvn5eWlu+66S1OmTNGKFSsIaQEAKCc88gcoZ3iLi4c4uJTVKh06lNOOXtYA4NR43B2AMzMd0s6ZM0cHDhxQixYtNHv2bPn6+mrZsmWXhbSS1KVLF02ZMkXR0dE2FQsAAEqGR/6A/5eQYN92AAAAzoQXpzoN0yHtTz/9JMMwNGLECPn6+hbZNjQ0VBaLRQcPHjS7OQAwhZ6EcFc88ufeOPZdJDjYvu0AAACcBS9OdSqmQ9oDBw7IYrGU6DEBi8WigIAAnT592uzmAKDU6EkId8cjf+6JY98lOnXKuRiJjy94XFrDyJnfqVP51wYAAFBWeHGq0zEd0qanp6tSpUqylLCLdFpamipVqmR2cwBQavQkBOCOOPZdwmLJ6S3Sp09OIHvxhUruC28nTuSxPwAA4Dp4capTMh3SBgUFKSEhQadPn1aVKlWKbBsXF6e0tDQ1bdrU7OYAwBR6EgJwRxz7LtG7d05vkYIe95s4kV4kAADAtfDiVKdkOqQNDw/X4sWLtWTJEvXt27fItp9++qkMw1D79u3Nbg4AAAAwr3fvnN4ibvDiDMYkBgDAzfHiVKdkOqS9//779eOPP2rKlCkKDw9Xs2bNLmuTnp6uyZMna9GiRfLw8NB9991nU7EAAAC5oqOjTS9bXEhVlNjYWNPbhYNZLC7fW4QxiQEAAC9OdU429aTNHc/s3nvvVadOnXT27FlJ0rvvvqv4+Hht3rxZycnJkqSBAwcy3AEAALBZZmamJGnAgAEOrSMgIMCh2wcKwpjEgHujJz0ASbw41UmZDmkl6aWXXpK/v78++eQT/fTTT5IkwzD0ySefSJKsVqs8PT01cOBADR482PZqAQCA24uIiNCWLVvk6WnuNKYkIVVxAgICFBoaampZoKwxJjHgnuhJDyAPL051SjaFtIZh6Omnn9bdd9+tBQsWKDo6WseOHVNWVpaCgoIUHh6uPn36KCQkxF71AgBQYvQmcV0RERE2r4OQCnBNHPvhruhJDyAfXpzqdEyHtEeOHJEk1ahRQ/Xq1dOQIUPsVhQAALaiNwkAuB+O/XB39KQHkI8bvTjVFZgOaW+55RZ5eHhozZo1ql27tj1rAmBH9CaBu6I3CQC4H479ANwZ134okBu8ONVVmA5pfX195eXlRUALVGD0JoG7ozcJALgfjv0A3BHXfoDzMx3S1qtXT3///beysrJkoZs0KjB3vptIbxIAAAAAcH1c+wHOz3RI26VLF3344Ydat26dbrnlFnvWBNgNdxPpTQIAAAAA7oBrP8C5mQ5pBwwYoCVLlujVV19V3bp11bx5c3vWBdgFdxMBAAAAAABQ0ZkOaX/66Sf17dtXkydPVp8+fdSxY0eFh4erRo0aRQ5/EBkZaXaTgCncTQQAwPVER0ebXra4oY6KEhsba3q7AAAAtuD8x7WZDmmHDx8uwzAkSVarVevWrdO6deuKXMYwDEJaAAAAmJaZmSkp56kuRwoICHDo9gEAgPvg/Mc9mA5p69ata886AAAAgGJFRERoy5Yt8vQ0dxpbkqGOihMQEKDQ0FBTywIAAJQW5z/uwXRIu3r1anvWAQAAAJRIRESEzetgqCMAAOBMOP9xfaZDWjiP/fv3F/riLKn4cUkCAwNLNK4rAAAAAAAAgNIjpHVxSUlJCg0NVXZ2tul1WCwWJSYmKigoyI6VAQAAAAAAAJDsGNLGxcVp9+7dOnHihCSpRo0auvrqqxmvwsGCgoIUFxdXaE/akoxLEhgYSEALAAAAAAAAlBGbQ9o1a9bo3Xff1Z9//lng/KZNm+rpp59W586dbd0UTCrJUAWMSwIAAAAAAAA4hoctC0+ZMkWDBg1SXFycrFarLBaLatSooRo1ashischqtSouLk5DhgzR5MmT7VUzAAAAAAAAALgM0z1pf/75Z02ZMkWS1K5dOw0cOFBt27aVt7e3JCk9PV3bt2/Xhx9+qK1bt+qDDz5Q69at1alTJ/tUDgAAAAAAAAAuwHRP2i+++EKS1LVrV82cOVPXXXddXkArSd7e3rruuus0Y8YMde3aVVarNW8ZAAAAAAAAAEAO0yHt7t27ZRiGRowYIcMwCm1nGIaGDx8uSfr999/Nbs5hfvnlFz3xxBP617/+pVatWqlr166aOHGizp075+jSAAAAAAAAALgA0yFtRkaGqlSpotq1axfbtk6dOqpataoyMjLMbs4hZs2apYceekhr165VpUqV1KRJE8XHx2vatGnq06ePUlJSHF0iAAAAAAAAACdnOqStX7++zp49q/T09GLbpqen6+zZswoJCTG7uXK3e/dujRs3TpI0evRorV27VgsWLNDKlSt11VVX6a+//tLIkSMdXCUAAAAAAAAAZ2c6pL3zzjuVmZmpRYsWFdt20aJFyszMVPfu3c1urtx98MEHys7OVs+ePXXvvffmDelQu3Ztvfvuu/Lw8NBPP/2kP/74w8GVAgAAAAAAAHBmpkPahx9+WG3atNGYMWO0YMGCQtstXLhQY8aMUdu2bfXII4+Y3Vy5Onv2rNavXy9Juueeey6b36hRI/3rX/+SJC1btqxcawMAAAAAAADgWjzNLvjRRx+pbdu22rdvn/773/9q8uTJioiIyBuj9ujRo9q6dasSEhIUEBCgNm3a6MMPPyxwXUOGDDFbRpmIjY1Venq6vL291apVqwLbtGnTRps2bdKvv/5q8/asVqvOpp8tcJ7Fw6LKnpXzvi+snSR5GB7y8fIx1fZ85vlC2xuGIV8v37zvz2Wck9VqLVHb8xnnlW3NLrQOP28/U23TMtOUlZ1lc9vzmefzfX8h84IyszMLXa+vl29er+ri2vp4+cjDyLkPkp6VroyswsdkLk3byp6VZfGwlLptRlaG0rPyD09yPvO85JXzNTM7U54enoW2vVglz0p5bTOzM3Uh80Khbb0t3vKyeJW6bVZ2ltIy0wpt62XxkrfFu9Rts63ZOp+R8/9+8f7n/vwX1rYgnh6equRZSVLO5/hcRuEvFCxN29J87m05RhS0/4W1Lc3n3tmOEReyLhT571aaz70zHSPOZ56XLnrvZ2k+985+jLj0Z780n3tXOEYU9tm35TzC2Y4R8iz63Kc0xxNnO0ZkWf+3L7aeR1zMWY4RuT//Gdn/22+z5xG2tnXEMeLSz395XWtUlGNEUec+ZXGtIVW8Y4QsRR//7HmtcbGKcIww+7l3lWNE7s//haz8/0Zlda1R0Y4RBX3+HZ1HSOV3jCjq+OeoPOJi5XGMkEfRx7/yyCOKY1gL++kuRvPmzfN+OHJXkft9rsKmXyo2NtZMCWXmu+++08svv6xGjRpp+fLlBbb5/vvv9cILLyg4OFhr164tcn2dO3cudF5CQoLOVz6vgz0PFji/W2g3Lb5/cd73fuP8Cj0439jwRq196H+11HyrppLOJRXYtm3dtto2YJuio6PVpk0bBb8RrITzCQW2bVGzhfYM2pP3/VUfXKWY4zEFtm1YtaEOPv2/fWn3STttP7K9wLZBvkE6/sLxvO9v+uImrft7XYFtfb18dfa///sg3fHlHVoSt6TAtpJkffV/P9Z3f3e35sbMLbStxko7ftmh8PBwPbTwIc34dUahTY89f0w1/WpKkgYvHqwPtn9QaNsDww6oUWAjSdILP72gtze/XWjb3QN366paV0mSXlv7mkatG1Vo262PbVW7eu0kSW9tfEsvrnyx0LZr+q/RTY1ukiRN3TpVQ5YWfkPkx/t+1B3N7pAkfbHrCz286OFC237b51vdfdXdkqTv9nyne+Ze3uM81/Se0/VQ64ckSYv3LVb3rwof9mTK7VM0OGKwJGntwbW6ecbNhbZ9s8ubeuH6FyRJ2+K3KeLTiELbvnrjq3rtptckSXuO7dHV064utO3zHZ7XW/9+S5J0MOWgrph0RaFtB7UdpKl3TJUkHT97XLXerlVo2/7X9NcXkV9Iyjlh8R/vX2jbPi366Lu7v8v73hhV+DHUlmNEtfHVlJKeUmDb3GNErkYTG+nvU38X2NZZjxG5x7/OH3XWqoRVhbZNHZGad6LlascIfSHtmJdz/OMY8ZokjhGlPY/I5UzHiOjoaLV5p43UrMCmkkp3HuFsx4iZnWbqwc4PaseOHVp1fpXbnkcMCxumifdMlMQxgmNE2V5rVKRjRHR0tNo810a6qdCmLn2t8Z+r/6M3+ryhHTt26HT10257HtE5uLNWPr4y73uOETk4Rrh2HhEdHa02fdtI/QptWi7XGsUx3ZO2Xbt2Zhet8E6dOiVJqlq1aqFtcufltgUAAAAAAAAAM0z3pHVlU6dO1fvvv6+2bdtqzpw5BbbZvHmzHnroIVksFsXEFHwXpyQ6d+4sq9WqH5b9UOD8sn68ILcn2YYtG9S6desC21bExxTt9XjBrl271LF9R+3YkdOTrCI8glSejxfs2rVLHTt21IYNG9S+bXuHP4JU3sMdXLz/uT//Ff0xRVvaXnqM2Lh142X7X1jbivKYoj2PEbnHv01bN6nVNQUPbSNVvMcU7XWM2LVrlzr+q6N2bM85/lWER5DK6xhx6WffWR5TzGXrMaKgY59U8R9TLIiZY0R0dLTatG+jDRsLP/epiI8p2usYEfN7jCLaRmjHjh26+pqrneoxRXscI3J//n/Z9Ivat21fZNtcrnSMuPTz70qPMhfk0mNE9M7oQs99nP1R5pIcI6Kjo9Umoo02bCr8+OfMjzIX13b3b7v1r3b/0o4dO3RN62vcbriD3M//po2b1KFdh7zp7jLcQUHnP47OI6TyO0Zs27Gt0OOfOwx3EB0drTbt2mjD5sKPfxVhuAPTPWldWaVKOQe3jIzCf9jS09PztbWFYRj5x0krQknblbatj6dPidtffNArdr0XHXjt2fbiXxS2tPXxzL/NSp6VVEkl+z8tTVtvi3eJP5Rl1dbL4vW/sVj+n4+nj5SR8zX3IFdY28J4enjK07tkh5LStLV4WEr8M1math6GR17bi/e/oOUvbluc0nyOS9NWKtvPfVH7f7HSfO6d7RhRyVKpxP9urnSM8PH0kS46vy3N597ZjxFF/eyX5nPvrMeIkn72S7NeZztGKLPk5z6lOp44wTHCYlhMrddVjhG5P/9eHl7Fti2Isx8jivv8l9Wxp6IcI0p6/LPXtcalKsIxQlklP/652jHC7OfeVY4RuT//lSz5f64qwnVJeRwjivv8OyKPuFRZHiNKevwrzzzCHm1Lc4xQdsmPf2V1jCiOh13WUkK5wWZFV5KhDEoyJAIAAAAAAAAAFMd0SPvKK6/owoXCuxRfat++fbrrrrvMbq5cNWrUSJJ05MiRQnvT/vPPP/naAgAAAAAAAIAZpkPab7/9Vn369NG+ffuKbTtnzhzdc889+vPPP81urlyFhYXJy8tL6enp+u233wpss2PHDkkqdCwLAAAAAAAAACgJ02PSVq1aVXFxcbr77rv14osvql+/fpe1OXXqlEaMGKE1a9bIarUqPDzcpmLLi7+/vzp27Kg1a9bo22+/VZs2bfLNP3jwoH755RdJUteuXR1RIgAAAAAAgNuIi4vTmTNnTC0bGxub76sZAQEBCg0NNb08UBzTIe2iRYv0wgsvaNu2bRozZow2btyocePGKTAwUJK0ZcsWvfjiizp27JgMw9CgQYM0ePBge9Vd5gYNGqS1a9dq0aJFCg8P1z333CPDMHTs2DE9++yzys7OVpcuXdS8eXNHlwoAAAC4DS7S4c74+Xdv7vz/HxcXp2bNmtm8nqioKJuW37dvH58BlBnTIW2dOnU0c+ZMTZ06VdOmTdOaNWvUs2dPjR07Vtu2bdOnn36qrKws1alTR2+99ZbatWtnz7rLXKtWrTR8+HBNmDBBr7zyiqZNm6Zq1arpzz//VHp6uq644gq9/vrrji4TAAAAboaLdC7S4Z74+Xdv7v7/n/t7b/bs2QoLCzO1juTkZFWrVs3UsrGxsYqKijL9+xcoCdMhrSQZhqEhQ4aoQ4cOeuGFF3TkyBENGDBAkmS1WnXrrbdqzJgxqlq1ql2KLW8PPfSQrrzySn3++ef67bffdOLECdWtW1ddu3bV448/Lj8/P0eXCAAAADfCRToX6XBf/Py7900q/v9zhIWFOc1QmkBp2RTS5mrTpo0GDBigUaNGyWq1yjAMNW/eXO+88468vb3tsQmH6dChgzp06ODoMgAAAAAu0v8fF+lwZ+768+/uN6lyuev/P+AObA5p09LSNHr0aC1YsECSVLt2bR09elR79+7VXXfdpffee09Nmza1uVAAAAAAObhIB+BuuEkFwNXZFNL+8ccfeuaZZ3Tw4EFZrVZFRUXpxRdf1Nq1azVy5EjFxcXprrvu0osvvqh+/frZq2YAAAC35s6PewIA3Bs3qQC4KtMh7YwZM/TOO+8oPT1dgYGBGjdunG655RZJ0r///W+1bNlSzz//vHbs2KExY8Zo48aNGjt2rOm7VgAAAOBxTwAAAMAVmQ5px48fL0mKiIjQW2+9pdq1a+ebHxwcrFmzZumDDz7QtGnTtGbNGvXs2VM///yzbRUDAAC4MR73BODOeJIAAOCqTIe0FotFQ4YM0ZNPPinDMAps4+HhoSFDhqhDhw56/vnnlZiYaLpQAAAA/A+PewJwNzxJAABwZaZD2tmzZ+vaa68tUds2bdpo0aJFGjlypNnNAQAAAADcGE8SAABcmemQtqQBba4qVapo0qRJZjcHAAAu4chHPnncE4DTycqS1q+XEhKk4GCpUyfJYnF0VTCBJwlQanz+ATgB0yEtAABwnIrwyCePewJwGvPnS8OGSYcP/29a/frSpElS796OqwtA2ePzT0gNOIkSh7QLFy5UpUqVdPvtt5va0Lhx45Samqpx48aZWh4AUHr79+9XSkpKgfNK0pMyMDBQjRs3LovSYCNHPvLJ454AnMr8+VKfPpLVmn96fHzO9Llz3Seogfty15COzz8hNeBEShzSDh8+XDVr1iwwpO3YsaNOnjypmJiYQpdfsmSJTpw4QUgLAOUkKSlJoaGhys7OLrJdUT0pLRaLEhMTFRQUZO/yYCc88gmgxNwxpMnKygknLg1opJxphiE9/bTUs6fT/Fs4cqgbieFunJK7hnQu+PkvNUJqwKmUargDa0EHtxLMA1B23P1E3d33vyhBQUGKi4srtCetVHxPysDAQAJawJW4Y0iHHO4a0qxfn3+fL2W1SocO5bS76aZyK8usijDUjcRwN07FnUM6F/v8lxohNeB0GJMWTs+dQzp3P1F39/0vCYYqAJDHXUM6uHdIk5Bg33YO5sihbiSGu3E67h7Sudjnv9TcPaS+GDep4SQIaeHU3D2kc/cTdXfffwAoMXcO6dydu4c0wcH2bVdBMNQNSsTdQzoX/fyXmLuH1Lm4Se0y3KGDHiEtnBohXQ53P1F39/0HgCK5e0jn7tw9pOnUKediPD6+4M+AYeTM79Sp/GsDypq7h3Tu/vl395Ba4ia1C3GXDnqEtC7AHe4mFIeQDnBPHP+AEnD3kM7duXtIY7Hk9Jbq0ycnkLn4Qt0wcr5OnMgNCrgmdw/p3P3z7+4hNTepXYq7dNAjpHVy7nI3AQAuxfEPKCF3D+ncnbuHNFJOL6m5cwt+3HXiRHpRwXW5e0gnuffn391Dam5SuyRX76BHSOvk3OVuAgBciuMfUEKEdO6NkCZH7945vaV4cQzcibuHdLnc+fPvziE1N6nhhEoV0l64cEELFy68bHpaWpokFTjv0jYoG65+NwEACsPxDygGIZ17I6T5H4uF3lJwP+4c0l3MnT//7hpSc5MaTqhUIW1qaqpGjBhR6Pyi5lmtVhm5J4IAAAAoH4R0IKQB3Ju7hnQuKjo62tyCVaooOStL1apUkX79tdSL2/IeB4fgJjWcUKlCWmtBP9gAAACo2Ajp4GIhjemQQrYPdQM4JXfuSeoiMjMzJUkDBgxwaB0BAQEO3X6JcZMaTqjEIe2qVavKsg4AAACUJRcL6WCCC4Q0hBQA3FVERIS2bNkiT09zrxbKfZ+CLe9zCAgIcK4X5nKTGk6mxJ/uevXqlWUdQPnLyuJCFQDgXlwgpIN7I6QA4M4iIiJsXofbvc+Bm9RwIubObgBnN39+wXfTJk3ibhoAAEAFRkgBACgVblLDSXg4ugCg3M2fnzMuzcUBrZQzoHifPjnzAQAAAAAAgHJCT1q4l6ysnB60Bb0Ez2rNGUD86adzHofg8QcAAAAAFQwvzgMA10RIC/eyfv3lPWgvZrVKhw7ltHP1xyEYkxcAAABwGrw4DwBcGyGtO3PHkC4hwb7tnBVj8gIA4NToSQeUkgtc+/DiPABwbYS07spdQ7rgYPu2c0a5Y/JeOuRD7pi8c+e69s8AAABOjJ50gAkudO3Di/MAwHUR0rojdw7pOnXKOSGLjy94XFrDyJnfqVP511YeGJMXgOQSvYkAd0VPOqCU3PnaBwDgVAhp3Y27h3QWS84d8z59cvb14n8Hw8j5OnGia+67xJi8AFyqNxHgruhJB5SQu1/7AACcCiGtuyGkywkh5s4tOKSYONG1QwrG5P0fehLCHdGbCG5i//79SklJKXBe7piqRY2tGhgYqMaNG5dFaQDKE9c+AAAnQkjrbgjpcvTunXPH3N1COsbkzUFPQrgjehPBTSQlJSk0NFTZ2dlFtouKiip0nsViUWJiooKCguxdHoDyxLUPAMCJ2BzSJiYmavr06dqwYYOOHDmiCxcuKCYmJm/+qVOn9NVXX8kwDD366KOmx8+CnRDS/Y/F4n53zN19TF6JnoRwX/QmgpsICgpSXFxcoT1pJSk5OVnVqlUrdH5gYCABLVyPOz5FxLWPS4qOjja9bHHH/6IU9QQGANiDTYnpxo0b9fTTTys1NVXW/w88jNxxPf9f1apVtXLlSu3Zs0dNmzZV586dbdkkbEVI597cfUxeehIilzteqNKbCG6EoQqAS7jrU0Rc+7iUzMxMSdKAAQMcWkdAQIBDtw8UhuGenJ/pkDYhIUFPPfWUzp49q1tuuUWRkZEaOXKkTp8+fVnbu+66S7t379a6desIaR3N3UM6uPeYvPQkhOS+F6r0JnI59CQCUCLu/BQR1z4uJSIiQlu2bDH9dG5sbKyioqI0e/ZshYWFmVpHQECAQkNDTS0LlCWGe3INpkPazz//XGfPntXtt9+u9957T5I0evToAtt27NhRkvT777+b3RzsyZ1DOuRw1zF56UkId75QpTeRy6AnEYAS4ykirn1cTEREhM3rCAsLU3h4uB2qgSNwk7pgDPfkGkyHtBs2bJBhGBo2bFixbUNCQuTt7a3DRfVgQ/ly15AO/+OOY/LSk9C9ufuFKr2JXAY9iQCUGE8R5eDaB3B63KQuHkMVOD+bhjuoXLmyGjVqVKL2vr6+Sk1NNbs5lAV3DOng3uhJ6N64UKU3kQuhJxGAEuEpov/h2gdwatykhjswHdIahlHsWBe5MjMzlZqaKj8/P7ObAwDb0ZPQvXGhmoPeRADgPniKCIAL4SY1XJ2H2QXr1aun9PR0HTlypNi227ZtU2ZmZol73QJAmcntSVivXv7p9eu79nik4EL1Yrm9ie67L+crAS0AuKbcp4hyb0ZfyjCkkBCeIgIAuK6sLGntWumrr3K+ZmU5uqJCme5J26FDB/3111/6+uuv9eyzzxbaLiMjQxMnTpRhGOrEL38AFQE9Cd0Tw124lf379xf64oTcFz8U9QKIwMBAxvUC4Px4iggA4M7mzy94qLdJkypkBy3TIe1DDz2kb775Rp9//rlCQkJ09913X9Zmz549Gj9+vH799Vf5+/vr/vvvt6lYoDDu/oZHd99/UxiXzP1woeo2kpKSFBoaWuywTFFRUYXOs1gsSkxM5A23AJwf45EDANzR/Pk5136XdtCJj8+ZXgGfpDUd0tarV09jxozR8OHD9corr+i9997TmTNnJEl9+/ZVfHy8kpKSZLVa5enpqTfeeEPVq1e3W+GAxBse3X3/gVLjQtUtBAUFKS4urtCetFLxN6gCAwMJaAG4Dp4iAgC4k6ysnGu+gp6gtFpzOuk8/XTO78YK9LvQdEgrST169FCNGjU0evRo/f3333nTd+3alff3hg0b6rXXXlOHDh1s2RSK4a49Kd39DY/uvv+AKVyougWGKgCAS/AUEQDAXaxfn79TzqWsVunQoZx2Feh3o00hrSRdf/31WrZsmbZt26bo6GgdO3ZMWVlZqlmzpsLDw9W+fXtZuPAtM/Sk5A2P7r7/gClcqAIAAACAa0pIsG+7cmJzSCtJhmEoIiLCLmERSoeelAAAAAAAAMD/Cw62b7tyYpeQFo5FT0oAAAAAAABAOcPZ1a+f85KwgsalNYyc+Z06lX9tRSCkBeD03HVMZgAAAAAAcAmLRZo0SerTJyeQvTioNYycrxMnVrj3ktgc0m7ZskWLFy/W3r17lZKSkjdGakEMw9DKlStt3SQASGJMZgAAAAAAUIDevaW5c6Vhw/K/RKx+/ZyAtndvh5VWGNMhrdVq1X//+18tXLgw7/viGLlpNQDYAWMyA47pSU4vcgCAI/EUFQCgRHr3lnr2lNavz3lJWHBwzhAHFawHbS7TIe2sWbO0YMECSdJVV12lW265RbVq1TIdlgCAGYzJDHdVEXqS04scQEW0f/9+paSkFDgvN2QrKmwLDAxU48aNy6I02Kgi/O6T+P2HiovjH1AAi0W66SZHV1EiphPV+fPnyzAM3X333Ro9erQ9awIAAMVwdE9yepEDqIiSkpIUGhqq7OzsIttFRUUVOs9isSgxMVFBQUH2Lg82cvTvPonff6i4OP4Bzs90SHvw4EFJ0nPPPWevWgAAQCnQkxwA8gsKClJcXFyhPcmk4h93DwwMJKCowPjdBxSM4x/g/EyHtJUqVVKlSpVUtWpVe9YDAECpMC4dAOBiPKoLwF1x/AOcm+mQtlmzZoqOjtbZs2fl5+dnz5oAACgW49IBAAAAAFyF6ZC2X79+2rZtm+bNm6cHH3zQnjUBAFAsxqUDAAAAALgK0yFt165d1a9fP7399tuqUqWKIiMj7VgWAADFY1w6AAAAAIArMB3SjhgxQpLk4+OjESNG6P3339fVV19d5NAHhmFo3LhxZjcJAAAAAAAAAC7HdEi7YMECGYYhq9UqSTpy5IiOHDlSYNvcdoS0AAAAAAAAAJCf6ZA2MjJShmHYsxYAAAAAAAAAcDumQ9oJEybYsw4AAAAAAGCD/fv3KyUlpcB5sbGx+b4WJDAwUI0bNy6L0gAAxTAd0gIAAAAAgIohKSlJoaGhys7OLrJdVFRUofMsFosSExMVFBRk7/IAAMUgpAUAAAAAwMkFBQUpLi6u0J60kpScnKxq1aoVOj8wMJCAFgAcxC4h7apVq7RhwwYdOXJEaWlpmjFjRt68c+fO6Y8//pBhGLr22mvtsTkAAAAAAHAJhioAAOdlU0ibkJCgIUOGKCYmRpJktVove5mYl5eXnnvuOSUmJurrr7/WNddcY8smAQAAAAAAAMCleJhd8Ny5c3rkkUe0Z88e1a5dW/369ZOPj89l7by8vHTXXXfJarVqxYoVNhULAAAAAAAAAK7GdE/aOXPm6MCBA2rRooVmz54tX19fLVu2TGlpaZe17dKli6ZMmaLo6GibigUAAAAAd2fLdVVxY5IWJTY21vR2AQBA0UyHtD/99JMMw9CIESPk6+tbZNvQ0FBZLBYdPHjQ7OYAAAAAwK1lZmZKkgYMGODQOgICAhy6fQAAXJHpkPbAgQOyWCwKDw8vtq3FYlFAQIBOnz5tdnMAAAAASmD//v2Fvt09tydkUT0iAwMDeflQBRUREaEtW7bI09PcZVxsbKyioqI0e/ZshYWFmVpHQECAQkNDTS0LAAAKZzqkTU9PV6VKlWSxWErUPi0tTZUqVTK7OQAAAADFSEpKUmhoqLKzs4tsFxUVVeg8i8WixMREBQUF2bs82EFERITN6wgLCytRZxsAAFB+TIe0QUFBSkhI0OnTp1WlSpUi28bFxSktLU1NmzY1uzkAAAAAxQgKClJcXFyhPWml4sckDQwMJKAFAAAoZ6ZD2vDwcC1evFhLlixR3759i2z76aefyjAMtW/f3uzmAAAAAJQAQxUAAAA4H9Mh7f33368ff/xRU6ZMUXh4uJo1a3ZZm/T0dE2ePFmLFi2Sh4eH7rvvPpuKLQ+nT5/Whg0b9Pvvv2v37t3avXu3zp07p3r16mn16tWOLg8AAAAAAACAi7GpJ23uoPP33nuvOnXqpLNnz0qS3n33XcXHx2vz5s1KTk6WJA0cONAphjvYunWrnnnmGUeXAQAAAAAAAMBNmA5pJemll16Sv7+/PvnkE/3000+SJMMw9Mknn0iSrFarPD09NXDgQA0ePNj2astBpUqV1K5dO7Vs2VJXX321UlJSNHr0aEeXBQAAAAAAAMBF2RTSGoahp59+WnfffbcWLFig6OhoHTt2TFlZWQoKClJ4eLj69OmjkJAQe9Vb5jp16qROnTrlfb9mzRoHVgMAAAAAAADA1dkU0uaqV6+ehgwZYo9VAQAAAAAAAIBb8XB0AQAAAAAAAADgzuzSkxZF69y5c6HzEhISFBwcXI7VAAAAAAAAAKhIShXSJiQk6K+//lLlypXVtm3bfPOeeuoppaSkFLrsf/7zH1111VWmigQAAAAAAAAAV1WqkPaFF17Qjh079Nxzz10W0kZHR+vEiROyWq2XLWcYhiZMmKBZs2bZVq2TWrVqVaHziuplCwAAYMb+/fsLvXkeGxub72tBAgMD1bhx47IoDQAAAEABShzSxsbGavv27QoODtYjjzxSaLtevXpdNm3t2rXavn279u3bp2bNmpmrtBhjx47VzJkzS71cRESE24bHAADA9SQlJSk0NFTZ2dlFtouKiip0nsViUWJiooKCguxdHgAAZYablACcWYlD2p9++kmSdO+998rDo/D3jY0fP/6yaZ988oneeecdLV68uMxCWl9fXwUGBpZ6OX9/f/sXAwAA4CBBQUGKi4srchiq5ORkVatWrdD5gYGBBLQAAKfCTUoAzq7EIe2uXbtkGIY6depU6o3ceuuteuedd/Trr7+WetmSeuaZZ/TMM8+U2foBAACcBb2AAADuhpuUAJxdiUPav/76S4ZhKCwsrNQbadiwoby8vLR///5SLwsAAAAAAFAcblICcGYlDmlPnTqlgICAQoc6uOWWW5SamlrgPMMw5O/vr9OnT5urErAB4xIBAAAAAACgIitxSCtJ6enphc4bPXp0kcumpaXJarWWZnOwE3cOKRmXyL3//wEAAAAAAJxBiUPawMBAHTt2TKmpqaV+2VZqaqrOnz+v2rVrl7pAR2jfvn3e3zMzMyVJCQkJ+aZ3795dI0eOLPfaSsvdQ0p3H5fI3f//AQAAAAAAnEGJQ9pGjRrp2LFj2rFjh2688cZSbWTbtm2SpCuuuKJ01TlIQYFednZ2vulnz54tv4Js4O4hpeTe4xLx/w8AAAAAAFDxlTikjYiI0JYtWzR79uxSh7SzZ8+WYRiKiIgodYGOsHfvXkeXYFfuHFKC/38AAAAAAICKruC3gBXgrrvukqenpzZs2KAvv/yyxBuYM2eONm7cKE9PT/Xu3dtUkQAAAAAAAADgqkoc0tapU0f333+/rFarXn/9db3++us6duxYoe2PHTum0aNHa8yYMTIMQ/fff7/q1Kljl6IBAAAAAAAAwFWUeLgDSXrhhRcUExOj7du368svv9Q333yjq666Ss2bN1dgYKCknPFc//jjD+3Zs0dZWVmyWq1q166dXnjhhbKoHwAAAAAAAG5u//79hb6PJTY2Nt/XggQGBjJcIByqVCGtl5eXPv/8c40aNUrz589XZmamfvvtN/3222+XtbVarTIMQ3369NErr7wiT89SbQoAAAAAAAAoVlJSkkJDQ5WdnV1ku6ioqELnWSwWJSYm8uJsOEypk1Nvb2+NHTtW/fv315dffqnNmzfr77//ztemYcOG6tChg+6//341a9bMbsUCAAAAAAAAFwsKClJcXFyhPWklKTk5WdWqVSt0fmBgIAEtHMp099ZmzZrptddekyRlZmbq1KlTkqSqVavSaxYAAAAAAADlhqEK4OzskqZ6enqqRo0a9lgVAAAAAAAAALgVD0cXAAAAAAAAAADujJAWAAAAAAAAAByIkBYAAAAAAAAAHIiQFgAAAAAAAAAcyC4vDgOAimr//v1KSUkpcF5sbGy+rwUJDAzkLaEAAAAAAKBMEdICcFlJSUkKDQ1VdnZ2ke2ioqIKnWexWJSYmKigoCB7lwcAAAAAACCJkBaACwsKClJcXFyhPWklKTk5WdWqVSt0fmBgIAEtAAAAAAAoU4S0AFwaQxUAAAAAAICKjheHAQAAAAAAAIAD2aUn7dGjR7Vv3z6dOnVKmZmZRbaNjIy0xyYBAAAAAAAAuIno6OhC5x0+fFhnzpwxve6AgADVr1+/wHlFvWzcnmwKaffu3asxY8Zo+/btJWpvGAYhLQAA5WT//v2Fjsmce6JR1AlHYGAgQ4YAAAAAcKjcDqEDBgxwaB0BAQFlun7TIe3+/fvVr18/nT17VlarVV5eXqpevbosFos96wMAACYkJSUpNDRU2dnZRbaLiooqdJ7FYlFiYiIvzwMAAADgMBEREdqyZYs8PQuPMcuyJ23u/NDQUNPrLwnTIe2UKVOUmpqqWrVqadSoUbrhhhsIaAEAqCCCgoIUFxdXaE9aSUpOTla1atUKnR8YGEhACwAAAMDhIiIiipwfHh5eTpWUHdMh7ZYtW2QYht544w116NDBnjUBAAA7YKgCAAAAAHAOpkPaM2fOyNvbW+3bt7dnPQAA2A1jsgIAAAAAnIHpkLZmzZo6efKkPDw87FkPAAB2wZisAAAAAABnYTqkvfnmmzVnzhzFxMSoRYsW9qwJAACbMSYrAADuh6doAADOynRIO3DgQC1ZskTjxo3T559/Lm9vb3vWBQCAzbjIAgDAffAUDQDAmZkOaS9cuKDx48frxRdfVK9evfTII4+oVatW8vPzK3K5unXrmt0kAAAAAAAF4ikaAIAzMx3Sdu7cOe/vp0+f1ssvv1zsMoZhKCYmxuwmAQAAAAAoFE/RAACclemQ1mq1lssyAAAAAAAAAODKTIe0q1atsmcdAAAAAAAAAOCWTIe09erVs2cdAAAAAAAAAOCWPBxdAAAAAAAAAAC4M0JaAAAAAAAAAHCgEg13sG3bNklS5cqV1bJly3zTSqtdu3amlgMAAAAAAAAAV1SikPaBBx6QYRhq3LixFi9enG9aaRiGoZiYmNJXCQAAAAAAAAAuqsQvDrNarcrOzr5sWmmUtj0AAAAAAAAAuLoShbR//PFHiaYBAAAAAAAAAEqHF4cBAAAAAAAAgAMR0gIAAAAAAACAAxHSAgAAAAAAAIADEdICAAAAAAAAgAMR0gIAAAAAAACAAxHSAgAAAAAAAIADeTq6AAAAAACAfezfv18pKSkFzouNjc33tSCBgYFq3LhxWZQGAACKQEgLAAAAAC4gKSlJoaGhys7OLrJdVFRUofMsFosSExMVFBRk7/IAAEARCGkBAAAAwAUEBQUpLi6u0J60kpScnKxq1aoVOj8wMJCAFgAAByCkBQAAAAAXwVAFAAA4J7uFtCdPnlR8fLzS0tLUrl07e60WAAAAAAAAAFyazSHtqlWrNGXKFP3xxx+SJMMwFBMTkzf/1KlTevbZZyVJEydOVEBAgK2bBAAAAAAAAACX4WHLwh9//LGGDBmi2NhYWa3WvD8Xq1q1qipXrqxNmzZp2bJlNhULAAAAAAAAAK7GdEi7a9cuvffee7JYLBoxYoR++eWXQgeY79Gjh6xWqzZt2mS6UAAAAAAAAABwRaaHO5g5c6Yk6YknnlD//v2LbJs7Ru3FwyAAAAAAAAAAAGzoSRsdHS1J6tevX7Ftq1evLh8fHx07dszs5gAAAAAAAADAJZkOaU+cOCE/Pz9Vr169RO29vb2VkZFhdnMAAAAAAAAA4JJMh7S+vr5KS0tTVlZWsW3Pnj2rM2fOKDAw0OzmAAAAAAAAAMAlmQ5pr7jiCmVlZWnv3r3Ftl25cqWys7PVvHlzs5sDAAAAAAAAAJdkOqS95ZZbZLVa9dFHHxXZLjExUe+8844Mw9Btt91mdnMAAAAAAAAA4JJMh7T9+vVT7dq19dNPP+nFF1/Uvn378uZlZGTo4MGDmj59unr37q1jx46pUaNGioyMtEfNAAAAAAAAAOAyDKvVajW7cGxsrB599FGdPHlShmEU2MZqtapWrVr64osv1LhxY9OFuqrOnTtLklatWuXgSgAAAAAAAAA4gk0hrSQdP35c7733nhYvXqwLFy7km+fl5aXu3bvr2WefVc2aNW0q1FW1bNlSWVlZCg4OdnQpAAAAAAAAAOwsODhYs2fPLrKNzSFtrvT0dO3evVvHjh1Tdna2goKC1LJlS/n4+Nhj9S6rbdu2Sk9Pd1iInZCQIEluGxKz/+y/xP6z/+y/u3HnfZfYf/af/ZfYf/af/XdH7D/7L7H/7L/j9r9cQ1o4J3cfboH9Z/8l9p/9Z//djTvvu8T+s//sv8T+s//svzti/9l/if1n/yv2/pt+cRgAAAAAAAAAwHaEtAAAAAAAAADgQJ5mFwwLCytVe29vbwUEBCg0NFQ33HCDevfurapVq5rdPAAAAAAAAAC4BNM9aa1Wa6n+XLhwQUlJSdq8ebPefPNN3XHHHdq+fbs99wUAAAAAAAAAnI7pnrQzZ85UfHy8JkyYoPPnz+v2229XRESEateuLUk6evSotm7dqqVLl8rHx0cjRoyQv7+/fv/9d82dO1dJSUkaNGiQfvzxR9WqVctuOwQAAAAAAAAAzsR0SNu0aVM999xz8vf319dff60rrrjisjZ33XWXBg4cqMcee0yTJk3S/Pnz1aVLF/Xv31/9+vXTwYMHNWvWLD333HM27QQAAAAAAAAAOCvDarVazSw4ZswYzZkzR59//rk6dOhQZNvNmzfr4Ycf1kMPPaThw4dLktatW6cnnnhCzZs318KFC82UAAAAAAAAAABOz/SYtGvXrlWlSpWKDWglqUOHDvLx8dGqVavyTfP09NThw4fNlgAAAAAAAAAATs90SHvs2DFZLJaSb8jDQ0ePHs373tvbW35+fkpPTzdbAgAAAAAAAAA4PdMhbZUqVXTu3DnFxsYW2zY2NlZnz55VQEBA3rSsrCylpqYqMDDQbAkAAAAAAAAA4PRMh7Rt27aV1WrVyJEjdebMmULbnTlzRiNHjpRhGIqIiMibHh8fr6ysLNWuXdtsCQAAAAAAAADg9DzNLjho0CCtXLlSe/bs0e2336777rtP7dq1U61atWQYho4dO6YtW7bo66+/VlJSkjw9PfXkk0/mLb9s2TJJOWEvAAAAAAAAALgrw2q1Ws0uvGLFCr344os6f/68DMMosI3ValXlypX1xhtv6LbbbsubPmvWLB06dEi9e/dW8+bNzZYAAAAAAAAAAE7NppBWkg4dOqQPP/xQK1as0OnTp/PNq1Klim699VY98cQTatCggU2FAgAAAAAAAIArsjmkvdihQ4d08uRJSVL16tUVEhJir1UDAAAAAAAAgEuya0gLAAAAAAAAACgdD0cXAAAAAAAAAADuzNMeK8nOztbBgwd16tQpZWZmFtm2Xbt29tgkAABAPqmpqbJYLPLx8Sn1sn/88YfOnDnDeQoAAAAAh7BpuINjx47p3Xff1fLly5WWllb8xgxDMTExZjcHAGXu7NmzWrp0qRYsWKA5c+Y4uhyHOHTokObPn69hw4Y5uhSgVJo3b662bdtq9uzZl80bMmSImjZtqqeffrrAZe+//37t2rWL8xS4lV27dikjI4ObEwDcGuf/2Lx5sxYsWKA333zT0aU4xIkTJ3ThwgXVrVvX0aWUSufOnW1a3jAMrVy50k7V2IfpnrRHjx7VPffco2PHjqmkOS/D38JRUlNTtWPHDmVkZOiqq65ScHBw3rxz585pzpw52r17ty5cuKArrrhCPXv2VPPmzR1YsX1MnTpV4eHh6tChg6NLqfA2b96s+fPna+XKlSW66eRqzp49qyVLlmjhwoWKjo6WJLcNaY8ePaqsrCynO0m52Pbt2xUWFiY/Pz9Hl1LuCjvXWLlypZKTk00t6+xOnjypSpUqlejngR7F7mXIkCE6efIkNyf+n7NepBblyJEj2rVrl/bv369Tp04pLS1Nfn5+CgoKUsuWLRUeHi4vLy9Hl1kmzp07p6ysLAUEBFw277ffftPvv/+u9PR0XXHFFbruuuvk7e3tgCrLz/nz5zV//nytX79e8fHxys7OVp06ddShQwfdfffdqlq1qqNLLHeuev6/detWrV27VpmZmWrVqpXuuOMOGYYhKednf/LkyfmufXv16qX7779fHh7uNRrm33//rQULFuj7779XQkKCJDl1SBsREaFrr71WH3300WXzxo8fr5CQEEVFRRW47NChQ52ys0J8fLwMwzB9Dp/7uahITIe0U6ZM0dGjR+Xn56dnnnlGnTt3Vq1atWSxWOxZH+xk/fr1+uabb3TgwAFVrlxZ7dq104MPPljkSair9Cpavny5Xn75ZaWmpkqSLBaLnnjiCQ0dOlRHjx5V3759lZiYmPfBXrdunWbMmKEXXnhBDz/8sCNLt9nkyZNlGIaCg4MVGRmpXr16KSQkxNFlVRgHDx7M+8WcmJgoKSekCQwMVPfu3R1cXfnYtGmT5s+fr1WrViktLS3vcxAaGurgyhynZ8+eOn36tFMf+6KiouTj46N///vfioyM5EaNG8rIyND777+v7777TqdOnZIkhYWFacCAAbr99tsLXW706NFO/7v/wQcftGl5wzA0Y8YMO1VT8bnqzYmvv/76snPfxx57TNdcc02hyzjrRWpBfvnlF02aNEm7du0qsl1gYKD69eunxx9/3GVCygMHDuiVV17Rjh07ZLVa1bBhQ7300kvq1KmT0tPT9cwzz2j16tX5lqlVq5beeecdtW3b1kFV28f8+fP15Zdf6plnntH111+fNz0uLk4DBw5UfHx8vs/8X3/9pU2bNmn69OmaNGmS0+9/Sbj6+f/48eM1c+ZMSTn7ZRiGvvvuO3366afauXOnHnvsMaWnp+e137Nnj2JiYrRlyxZNnjzZUWWXm9TUVC1dulTz58/POz5arVZ5enqqY8eOji3ORqdPn87LPC41Y8YMtWnTptCQVnLu84EmTZooMjLSJa5hTYe0P//8swzD0NixY9W1a1d71gQ7mzZtmt5//31J//vgxcTE6KuvvtKLL76ofv36FbqsM39QJWnfvn167rnnlJmZKR8fH9WpU0eHDh3SBx98oKZNm2ru3LlKSEhQ48aN1b59e0nSli1btH//fr311ltq3bq1rr32WgfvhW2sVquOHDmiadOmadq0aWrbtq169eqlrl27ytfX19HllbvU1FQtXrxYCxYs0K+//iop59/Iw8NDN998s3r16qUbb7zRZXuVSDkXLwsXLtSiRYt09OhRSTn/BtWqVVP37t3Vq1cvtWjRwsFVOpazH/uknB4z33//vb7//nsFBwerV69eioyM5EaNmxg6dKjWrVuX72c5JiZGzz77rJYuXarx48cX2rPW2X/+t27d6nK9KkqjNI/+nTx58rJlKuKjf6X16quv6ttvv837GUhLS9OKFSu0atUq9e/fX88//3yhHUuc/edfkj7++GO99957Be6LYRi68sorlZ6ern/++UfJycmaOnWqli9frunTpysoKMgBFdtPcnKyHnjgAZ04cSJv/w8ePKjBgwfryy+/1Lx587Rq1Sp5enqqYcOGknJ60x09elRPPPGEfvjhB6fuSb18+XLFxMQoLCwsb9q5c+c0YMAAJSYmqnLlyurWrZuaNGmiSpUq6Z9//tGSJUuUlJSkJ598UgsXLlT9+vUduAdlw13O/9euXZt3k7FDhw5q0KCBdu7cqa1bt2rGjBlauHChMjIydNddd+UFkuvXr9eCBQu0cuVKLV68WHfccYcjd6FMWK1Wbdy4UQsWLNCqVat04cKFvOND8+bNFRkZqR49eqh69eoOrhSl1b17d61atUp//fWX3n33XYWFhal379664447VK1aNUeXZ4rpkPbkyZOyWCzq0qWLPeuBnUVHR+v999+X1WrV9ddfr06dOunChQtasmSJ9u7dqzFjxiguLk6vvfaao0stE9OnT1dmZqbuvPNOjR07Vt7e3jp27JieeOIJvf/++/r777/Vs2dPjR07Vp6eOR+HzMxMjRgxQj/88IPmzJnj9CFts2bN9O9//1sLFy7U4cOHtW3bNm3fvl2vv/66unbtqsjIyLyA2lVZrVZt2LBBCxYs0OrVq/P9Yr766qu1e/duVa9eXVOnTnVwpWXnzJkzWrx4sRYuXJjv5NTLy0sZGRmqXr26fv7557zPAZxf3bp1Vb9+fW3btk1HjhzRBx98oA8++EBt27ZV7969ddttt7nljRp38MMPP2jt2rXy9vbW4MGDdcMNNygtLU0//vijvvnmG61YsUIJCQn69NNPXfrx1iZNmuiOO+5w6X0siJlH/+Lj4/P+7uwh9bp16/TNN9/IMAzdc889+X7+165dqy+++EIHDhzQ+++/7zI9Ry+2efNmvfvuu/Ly8lJUVJS6deumOnXqKDExUUuXLtWsWbNUqVIlffvtt5JyOt5MnjxZe/fu1YABAzR37lynfjLy888/V1JSkpo3b66RI0eqYcOG2r59u0aNGqX33ntPO3bs0NVXX62JEyfmhZGHDh3SsGHDFBsbqxkzZmjEiBEO3gvz9u7dq1q1auULm+bPn6/ExESFhYXpo48+Uq1atfIt8+yzz+rll1/Wjz/+qE8++USjRo0q77LLhDue/3/99dcyDEPDhw9X//79JeW85P2ZZ57RRx99pNTU1HzzJOn2229Xs2bNNGHCBC1YsMClQtr9+/fn9Zq+eJjOoKAgJSUlKSgoSAsXLnRskbDJ22+/nTem9MKFC7Vjxw6NHTtWb7zxhm644Qb16tVLN910k1Nd45qutEaNGkpNTXWqnXVHc+bMkdVq1cMPP6z//Oc/edMff/xxzZw5U2+99Za++eYbnTt3ThMmTHC5cWi2bt0qHx8fvfrqq3kn4rVq1dJ//vMfPfTQQ6pUqZJefvnlfD/Hnp6eGjlypJYvX543NqczCwgI0JAhQzRkyBBt27ZN8+fP1/Lly3Xu3DktXLhQCxcuVN26dfN62bnS3fO//vor7xfz8ePH834xBwcHq0ePHurRo4eaNGniEuMPF8RqtebdHV+9erXS09Pz/g1at26tnj17qlu3bmrfvr08PDxc6nhuywnXxY+AObPg4GDNnDlT8fHxWrBggRYtWqRDhw7l3agZPXq0unbtql69eikiIsLR5cKOFi5cKMMwNGbMGPXo0SNv+rXXXqtevXpp6NCh+v333/XAAw9o+vTpqlGjhgOrtb9rr71WO3fu1F9//aUPP/xQN910kyIjI3XjjTc6dfhUWl26dCm2V+3YsWN19uxZjRs3rpyqKnu5Ae2LL76Yb9iq7t27a+XKlRoxYoTWrVunxx9/XNOmTZOPj48Dq7W/GTNmyDAMvf3227rtttvyptesWVMtW7ZUs2bNNHz4cH388ccaMmSIunTpohtuuEGPP/64tmzZou+++059+/Z14B7YZu3atfL09NTkyZPznhzp2rWrLly4oP/85z+yWCx666238p3vhoSE6K233tKdd96pjRs3Oqp0uzh58qSuvPLKfNO2bdsmwzD02muvXRbQSlLlypU1ZswYrVmzRuvXry+vUsuMO5//7969W1WrVs037I+Hh4cGDx6s5cuXKyAgoMAhgR588EFNmTJFsbGx5VlumTh9+nRer+nff/9dUs41UeXKldW5c2f17NlT119/va666ioHVwp78fPzU58+fdSnTx8dPnw4L+NYtWqVVq9erapVq6p79+7q0aOHWrVq5ehyi2X6irxDhw5auHChDh48qEaNGtmxJNjTjh075OPjo2eeeSbfdMMw1L9/f7Vo0UKDBw/WDz/8oLS0NL377rsuFdQcP35cV1xxhfz9/fNNb9mypSSpQYMGBb5MoEqVKmrYsKH+/vvvcqmzvLRr107t2rXTK6+8kvcG0+3btys+Pl5Tp069rJedM1+43H333dq9e7eknF/Mfn5+eeNzunrPYUl666239P333yspKSnv5LR+/frq2bOnevbsqQYNGji4wrI1fPhw073BcsfvchX16tXLd6Nm3rx5Wr58uc6fP593ElOvXr28cavr1avn6JJho5iYGFWtWjVfQJvr6quv1nfffacBAwYoNjZWUVFR+uKLL1S7dm0HVFo2vvrqK/3zzz95F+k//fSTVqxYkW9Il4sfBXY1H330kV577TWtWrVKhmHo5ZdfLvT/95133tHZs2fVq1evcq6y7Pz222/y9/fP11MsV5cuXXTFFVfoscce05YtW/Too4/q448/vuw80Zn99ttvCg4OzhfQXiwyMlJvvPGGfvjhBw0ZMkSS5O3trVGjRum2227Tjz/+6NQh7eHDh1W/fv3Lhvbp1KmTpJyg7oorrrhsuSZNmqhu3br5epU7I19fX509ezbftNxxyYsayqpy5cpq3Lix9u7dW6b1lTV3P/9PSUnRlVdeedl5bO7QHiEhIQWe43p4eKhBgwbat29fudRZVp5++mmtWbMmr2OKYRhq166dIiMjddttt7nlC3XdTf369fOue7Zv364FCxZo+fLlmj17tubMmZN3DtC7d29Hl1oo090mn3zySfn4+Ojtt9+2Zz2ws6SkJDVq1KjQx7natWunmTNnKjAwUCtWrNDgwYNdpheZlPMLJyMj47LpudMKmndxG1cKai7m4+Oj3r17a9asWXn/7/Xq1VN2dra2bt2qESNG6Prrr9d///tfR5dqWu6d0ypVqmjcuHHatGmTxo8f7xYnaJL02WefKSkpSX5+frrnnns0Z84crVy5UkOHDnX5gPZizZo1U4sWLUr1x5VuVF2qXbt2mjBhgjZu3Kjx48fnvSDk8OHDmjp1qm699VY9+OCDPPrl5M6cOVPkUxFBQUGaNWuWrrnmGh04cED9+vVz+mDiUg0aNNCwYcO0atUqzZgxQ5GRkUpLS9PMmTPVu3dv9ejRQ9OnT9fx48cdXard3XjjjVq8eLH69eunVatWqVu3bpo1a5ZLjLVaEikpKWrQoEGhT4c1adJEX375Zd5YjQ899JBOnz5dzlWWnTNnzhQ7rmzt2rXz3mSeq2HDhmrYsKHi4uLKsrwyl5GRUWAng8qVK0tSkYG8n5+fMjMzy6y28tCkSRMdPHhQSUlJedPq1KkjSTp27FiRyx4/ftzpb1i4+/l/pUqVlJKSctn03GnJycmFLpucnKxKlSqVUWXlY9myZUpPT1eVKlX07LPPavXq1Xm/9wlo3U/btm01duxYbdy4UW+//baCg4N14MAB/fjjj44urUimr0QbNmyoadOm6amnntLDDz+sJ554Qq1atWJ8uwrG29u72JON5s2ba/bs2erfv79+/vlnPfnkk/rggw/KqcKyFRwcrEOHDuno0aP5epFs2bJFUk4wcfz4cdWsWTPfcseOHdPhw4fdokdZ/fr1NXToUA0dOlRbt27NNxzCggULnPoRSKvVqtOnT2v8+PHauXOnevTooXbt2jm6rHJ1/vx5xcfHKz4+Xi1atHDq3tGl0bBhQ/3zzz965ZVX1KZNm1It+69//Suv14mr8vX1Va9evdSrVy8dPnw4bziEw4cPa+vWrdq+fbsiIyMdXaYp+/btK/BRvpLMcxV+fn7Fhk7+/v6aPn26Hn/8cW3fvj2vR60rat++vdq3b69XXnlFy5cv18KFC7V161a98cYbeuedd3TdddepX79+uvHGGx1dqt34+vrq5ZdfVvfu3TVy5EiNGzdOixYt0ujRo13+xZA+Pj46d+5ckW2Cg4Pzzn13796dN/SHK6hZs6YOHjyo9PT0AjtpXLhwQQcPHizwSbKAgIDLwltnExQUpL///ltpaWl5wawk/fHHH5JyXhJW0L9N7ovUnPVFM7nuuOMO7dixQ++8847Gjx8vKWeoj4ULF2ratGkaM2ZMgcvNmzdPiYmJuummm8qx2rLhzuf/jRs31u7du/Xbb7/le6w7N5RKTEzUnj17LnvUf/fu3UpMTLxsqAxnlPv/v2DBAmVnZ+vOO+90i2t6FOzo0aNatGiRFixYoCNHjkhShe+QY7q6ix8T++WXX/TLL78Uu4xhGIqJiTG7SZjQoEED/fXXX7pw4UKRd8aaNGmSd7K6efNmPfbYY8We4DqD66+/XrNnz9Yzzzyj1157TSEhIdq5c6fGjRunpk2bKiUlRSNHjtTbb7+dd+c4NTVVL7/8srKysvJ6mbmLiIgIRURE6JVXXtGyZcu0YMECR5dk2k8//aR58+bphx9+0JEjR/Tdd99p7ty5Cg4O1p133pk3HpWrev3117VgwQJFR0dr48aN2rRpk1599VX9+9//Vo8ePXTddde5bE9xKeeR7n/++Ue7d+8udUjrbgq7UeOszpw5o61bt5Z6nuT8L0zK1aRJE+3cuVMnT54s8k3Fvr6++vTTT/Xkk0/ql19+UVRUlEu+SCmXj4+PIiMjFRkZqcTERC1atEhfffWV1q9fr8zMTJcKaXO1bt1aCxYs0IcffqiPP/5Yd999t6KiojRs2DCX7VjRqFEjxcTEKDU1tchegTVr1tTs2bP10EMPad++fXrggQdcordx+/bttXDhQr355pt66aWX8h3XrFarxowZo7S0tLw3u18sMTHR6d9ufu2112rZsmWaMGGCXnnlFXl4eOj06dN68803VaNGDZ07d05Tp069bCi4KVOm6Pz58+rQoYODKrePe++9V99++60WLlyo8+fPa+jQoerYsaP69eunL7/8Uv/884/69++vJk2ayMvLS4cOHdKCBQv0448/5g2H58zc/fz/1ltv1e+//64nn3xSgwYNUkhIiKKjo/X555/ruuuu0/Hjx/Xss8/qzTff1DXXXCNJ2rVrV967a5z99+D06dM1b948rVq1SgcOHNCkSZM0adIkhYeHKzIyUl27di3wBpUr+fvvvwt9+WFx81xFWlpa3k35LVu2KDs7+//au/Oopq61f+DfExBBBRxwwBbEgYo4DxRnxLkqEKhDlYtz1Sq8fW0darVYvY69Wq1DtWrrANjXKQk4gAi2WAuCgKg4FbUICmIvlBmEwPn9wS+5cgkIgeSQ7OezVlfXysmB5/GEnX322fvZMDAwwPDhw+Hu7o6xY8cKHWKNOF7N3og6hbY5jtOLYtS6ZP369Th9+jR2795dbW2qN6WmpmLu3LnKpwwAdPqapaenY8qUKSoHnLds2YKUlBQcOHAArVu3Vj5tvHPnDv7++28YGBjg3LlzOv1E0c7ODgMHDkRAQIDQoQiG53ncuHEDEokEYWFhKCoqUt6w2Nvbw8XFBdu2bYOFhQWuX78ucLQNLyUlpVJnFahoiy0sLJQF1N3d3fUu/2PHjmHbtm2YPHkydu7cWadzHR0dkZubq9NtX33/9gsLC3VyAGffvn31/hmKGo26bOfOnThy5AjWrVsHT0/Pt76/pKQE3t7euHbtmrJ91OXP/9uUlJTgypUrkEqliIyMRHl5OYYOHYqffvpJ6NA06smTJ1i3bh1u3boFS0tLrF27Fl9//TUyMzP16npv2bIFfn5+2Lx5c61qzuXk5GDevHm4f/++Xnz+nzx5Ag8PD5SUlKBbt24YO3Ys2rdvj4yMDISFheHx48fgOA5+fn6VHmKmpKRg/PjxGDlyJA4dOiRgBvVz69YtzJo1CwDQunVr5fLWwsJCLF68WPmAZtiwYRg6dCgAKB9mA8D3338PZ2dnweJvCP/+978xb948JCUlgeM4tGnTBh07dsT9+/dRVlam8hye57F8+XIsXrxYy9FqBqv9/+LiYnh4eODp06eVHtA0adIEJ06cQEpKClatWgWO45T9vMLCQvA8j9atW+PChQs6/6AGqJh0pdh/RbEROMdxaNKkCUaNGgVXV1d4e3vr3fW3s7MDx3F1fuCoOEfXx+uio6Mhk8mUe2/wPA9bW1uIxWK4urpWWT3dWKk9SFvTTJSa0A7S2hUWFgZvb28MGjQI/v7+tTrn5cuXmDt3LpKTk3X+DxWo6KytXr0aKSkpACpqUi1duhSLFi2CXC7HsmXLEBERUekcIyMjbNy4UWeX+yrQIG1lBQUFuHTpUpUvbOA/O9uOHTtW5+sxVScqKkplZ5XneZibmyMoKEhvNg+6e/cuvvrqK1hbW2PPnj11OvfIkSMoLi7W6cE6+ttn261btzBz5kxYWVkhJCQEBgYGbz1HLpfjs88+Q2hoqF5896sSGxsLmUyGkJAQFBQUgOd5dOnSBWKxGG5ubnrT/r1NQEAAdu3apfw30LfrHRkZifnz58POzq7W9bXz8/OxaNEixMfH68W/R2hoKFatWoXi4uIqM2kNDAzw5ZdfVnmAc+7cOfj5+cHLywsffvihtkNuUH5+fti+fXulkm/Dhg3DwYMHkZubi6lTpyI9Pb1SPwiomIW6YcMGQWJuaCUlJTh48CBOnjypskbpm/r06YNPP/0Uw4YN005wWsZa/z8rKws7duzAL7/8guLiYtjb2+PTTz9VjsMcOXIE+/btQ3FxsfIcW1tb7NixQ6cnJ1UnNTUV586dQ1BQUKUJKzzPo0WLFjhy5Aj69esnbJANpLpZsnWhKJOiK5KTkyGTyRAUFIT09HTwPK/cKFYsFlcp7aEL1B6kJbqhpKQEBw4cAM/zmDdvHszNzWt1XlZWFnbv3o3S0lKd+0OtzuPHj1FcXIzOnTtXKRz+yy+/IDY2FgUFBbCxscEHH3ygFzdrMTExMDU11etdrNWVmpoKiUSCwMDASl/YzZs3xwcffAA3Nze9LXdRUFBQ6emy4iZdJBJh8ODBEIvFGDduXKVabkS37Nu3D5aWljp/o03UFxUVBaDi5ru2m2WUl5cjODgYJSUlcHd312R4WpOamorAwEBlzWXFQ6nJkydDLBZXqtnHkoyMDPzzn/9UliG7evWqwBE1nPLyckgkEvA8jwkTJsDMzKxW5yk2llPMLNd1z58/h7+/P+Lj45GTkwNTU1P07t0bH330kV4OxPy31NRUXLt2Da9fv0aPHj0qlTHIzs7GwYMHERcXp+z7u7m51WrVoa4pKSlBXFwcEhMT8ddff6GwsBBNmzaFmZkZunTpgv79+9e40aS+Ybn//6asrCzcvXsXBQUF6Ny5MzP3iorZ1VeuXEFRURGAiutvbW0Nd3d3uLq6omPHjgJHSepCMXvY0NAQzs7OEIvFcHJyqtUEhcaKBmkJIcy7ceMGpFIpQkNDlV/YIpGIiRrais5qUFCQcnd3RWc1NjZW4OgIIaTuFMscZTKZ8kGUoaEhRo4cCbFYDGdnZzRp0kToMAkhhAiI5f4/6woLC3Hp0iUEBgYiNja20oSVe/fuCR0eqQPFIO27775b6wmJb+I4DmfOnNFAZOqjQVpCCPn/CgsLERISAolEgri4OJ1f7lhX0dHRkEgkCA0NRXFxMXP5E/0TGxuLixcv4tatW3j58qVyI6EOHTqgf//+mDx5MhMzZljTt29flJSUgOd52Nvbw93dHZMnT9aLOnuEEEIaFuv9f9YpNs+TyWRIT0+n669j1Nkr602NscRRgw3SZmZm4uXLl8oCvdVxcHBoiF9HCCEa9fz5c6aWf72psLAQly9f1pvlzoQ9GRkZWL16NaKjowFAZb9EUY/OwcEB33zzDTp06KDVGInmKGZVdOnSBd26davz+RzHYffu3Q0fGCGEkEaNhf5/amoqjhw5gjt37qCkpAQ2NjaYPn06nJychA5NcDExMbSHko6RSqX1/hmN7Z633oO0/v7+8PPzU27KVOMv4zhaPqBDNm3ahMLCQmzZskXoUOqtpKQEv/32G/7880+YmJhg4MCBb33q8sMPPyA5OVkvavLevXtXuZSnV69ecHV1hUgkAgDcvn0be/bsQWJiIsrKymBvb485c+ZgzJgxAkdNSMPIycnBs2fPYGJigm7dulXaREWV3377DZmZmTq/caACa/mnpqZi5syZyMzMVO7q6ujoCGtra5iYmKCoqAgpKSmIjo5GUlISAMDCwgI///wzrKysBI5eeB4eHsjNzUVYWJjQoahNH2dVaIs+XH8F6vuw1/7/N9bzryt9uvdjVUxMDL766it0795d5ca5d+7cwfz585UbRypwHIelS5fCx8dHm+GSBlbf7zCO4/Ti+1/X1WuQdvny5QgJCalx5ux/e/jwobq/jmjZ4MGDkZOTo/M3Kg8fPsSyZcuUxeEVRowYgQ0bNsDS0lLlebNmzcKtW7d0Pv9Dhw5h9+7dlf5OBwwYgKNHj+L27duYP38+SktLK53DcRxWrFiBBQsWaDvcBrN//34MGDCg0kYRhC15eXlYv349QkNDUVZWBgBo2bIlFixYgHnz5lVbUF5f/vZZzL+8vBxubm5ISkpC586dsWHDhhpnRMTExMDX1xfJycmwtbVFYGCgchCHVfrw3b9v3756/wx92DhKHfpw/QF2+z4KLLb/b2I9f3Xpy9+/wtOnT5WbRpuYmChf53keFy9exL179yCXy2Fvb4+JEydWeo+u2r17N3744Qd8/fXXmDFjRqVjZWVlmDx5MpKTkyESiTBu3DhYW1sjPj4ecXFxEIlEOHPmDHr27ClQ9Nrxxx9/4NmzZ5DL5ejQoQN69+4NQ0NDocNqEIqVROoO8bH8kLoxUfvTePHiRQQHB8PU1BSbN2/GyJEj0a9fP1hYWODatWv466+/EBkZiYMHDyI3Nxe7du3C4MGDGzJ2Qt4qKysLCxYsQGZmJgwNDdG1a1cUFxfj2bNnuHbtGsRiMfbu3au3yxoSEhKwa9cu5Wyyzp07IzExEfHx8Th27BguXrwIuVyOGTNmwNnZGXK5HJcvX8b58+exa9cuODk5qbVUtDHYu3cvOI6DpaUlxGIx3N3dmZwl99tvv+HUqVP4888/YWxsDAcHB8yePbvGnUtnzZqFhIQEnV75IJfLMX/+fCQmJlbqqPz999/YuXMnwsLCsGfPHrRr107AKDWH1fyDgoKQlJSEbt26wd/fHy1btqzx/e+//z5OnTqFmTNn4vHjxwgKCmJ2BpU+YXWAlVRgue8DsNv+K7CePwEePXqEFStW4PHjxwCAZs2aYcWKFZg5cyby8vIwZ86cKgNRe/bswcGDB9G9e3chQm4wcXFxAICxY8dWOXb9+nUkJyeD4zjs2LEDkyZNUh5bt24dzp49q/ODtIpVs6pKWEVERGDLli1VVoCbmZnhk08+wdy5c7UUpeZ17doVYrEYtra2QodC1KD2IK1EIgHHcfj0008xfvz4SsdEIhHat28Pd3d3jB8/Hl5eXli2bBkkEgk6depU76AJqa3jx48jMzMTPXv2xN69e5UDU4mJidiwYQPu3r2Ljz/+GHv27NHLOjz+/v7geR5eXl5Yu3YtgIrSD5988gl+/PFH5ObmYsmSJfj000+V54wdOxYWFhY4evQoTp06pTxPF/E8j7S0NBw4cAAHDhzAoEGD4O7ujokTJ6JZs2ZCh6dxBw4cUC51Utyo3L9/Hz///DNWrVoFT0/Pas/V9T0lT58+jbt376Jly5ZYt24dRo4cieLiYly4cAEHDhxAQkICPD09cezYMbzzzjtCh9vgWM0/JCQEHMdh/fr1bx2gVTA3N8fXX3+N2bNnIyQkRC8Gaf975Uhd6PrfPqHrz3rfh9X2X4H1/FmXnZ2N+fPnIzMzU/laQUEBNm7cCCsrK8hkMty/fx+tWrWCg4MDysrKEBMTg/T0dCxZsgQXL17U6XuE9PR0WFpaok2bNlWORUREAADs7e0rDdACgI+PD6RSqXKQV1d98MEHGDRoEPz9/Su9HhQUhC+++ALl5eUAKvp+TZo0QWZmJnJycrB9+3akpaXhyy+/FCLsBjNlyhSEh4fjyZMn+Pbbb9GjRw94eHhg8uTJaNWqldDhaUR9+jwKNU1eEoLa5Q6GDh2Kv//+G9HR0TAzMwNQMb26TZs2+P333yu999atW5g5cyamTp2KTZs21T9qUmv1qUuSnp4Onud1esq7m5sbHj9+jPPnz6NLly6VjsnlcmzcuBGnT59GkyZNsGPHDkyYMEF5XB+WPI0ePRpZWVm4fv06WrRooXw9Pj4es2bNQtOmTREZGYnmzZtXOi8nJwdDhw6Fra0tZDKZlqNuGHZ2dnjvvfcwfvx4yGQyPH/+HEDFMg5jY2NMnDgRYrEYjo6OAkeqGfHx8fD09ATP8xg2bBhGjBiB169f49KlS3j06BE4jsOMGTPw9ddfVzlXHz77np6eiI+Pxw8//ICRI0dWOpaWloZly5bhwYMH6NChA44ePYrOnTsrj1P+upv/yJEjUV5ejuvXr9f53OHDh0MkEuHatWsaiEy7FMvd1MHzPC1303GsX3+W+z4Au+2/Auv5s37v99133+HAgQN47733sGnTJnTt2hWxsbH48ssv8e677+LBgwfo27cvDh48qGwDsrOzsWDBAty/fx9r1qzB7NmzBc5Cff3794etrS1Onz5d5ZiHhwcePHiAZcuWqVxxMnr0aOTk5Oj0QK2dnR0GDhyIgIAA5WtZWVkYM2YMioqKMGrUKKxatUo5LpCVlYXDhw/j2LFjAAA/Pz8MGjRIiNAbTEFBAYKDgyGTyZTX0tDQECNHjoS7uztGjRqlN+UdAKBHjx71Or8x7pulduG13NxcNG/eXDlAC1Rc/MLCwirv7d+/P0xMTBAZGanuryNqevHiBdLS0vDixYs6/6d40qTLUlJS0LFjxyoDtEDF53Xjxo3w9vZGaWkpPv/8cwQGBgoQpeb89ddfeOeddyrdpADAe++9B6DiqdF/36QAFU8XO3XqhBcvXmglTk0xNTWFt7c3wsLC4OfnB3d3d+XGQTKZDHPnzsWYMWOwb98+5SCuvggICADP85g3bx5+/PFHzJ07F4sXL4ZMJsOaNWtgYGCAU6dOYdWqVXrxt/7fkpKSYGFhUeUGDaj43P/8888YNmwYXr58CS8vL/zxxx8CRKk5rOb/999/V1tn/G0sLS3x999/N3BEwuF5Xq3/iH5g+fqz3vdhtf1XYD1/1u/9IiIiwHEcdu7ciT59+qB58+ZwcnLC559/jtu3b0Mul+Orr76q1Aa0bNkSGzduBM/zuHr1qoDR119ZWRny8/OrvF5aWqrcLLVPnz4qz7WwsEBxcbFG4xNCUFAQioqKMGTIEBw8eLDSuEDr1q2xevVqLFmyBDzP48yZMwJG2jCaN2+OqVOnwt/fH1euXMGyZcvQvn17hIeHw8fHB8OHD8emTZtw584doUNtEOr2dxT/NcZ2T+0h9JYtW+L169eVXjMzM8Pff/+N3NzcSoO3Cv/+97/V/XVETc2aNUNRURHWr19f5xvXFStWoKCgQEORaYdcLlf5WXyTt7c3mjVrhm+++QZr1qxBSUkJpk2bpqUINatJkyZo2rRpldcVNy6tW7eu9lwzM7MqNXt0mYODAxwcHODr64vg4GBIpVLExsbixYsX2L9/P77//nsMGjQIHh4emDBhgs5vHhAXFwcTExMsX7680uscx2HOnDmwt7fHsmXLcP78eRQXF+Pbb7/Vq6eqhYWFNZbXMTY2xsGDB/G///u/CA8Px+zZs/Hjjz/qdB2uN7Gaf7NmzVTenNRGXl6eTi9xfJOFhQUyMzNx8uTJOtXX43keY8aMQU5Ojgaj0x5Wa3Kzfv1Z7/uw2v4rsJ4/6/d+ycnJsLS0rFKLc/jw4QAq2kfFA5s39ezZE23btlXWsdVVbdu2RXp6OoqLi2FsbKx8/e7duygtLYVIJKp2kLagoEDn739USUhIAMdx+OSTT6p9z6JFi/DTTz/p9CxiVd599114e3vD29sbsbGxkEqluHz5Mvz9/REQEIDOnTtj4cKF8PDwEDpUtYWHhwsdQoNT+468ffv2uH//PgoKCpRPohTLCaKjozFu3Djle+/du4eioiKYm5vXP2JSJz179kRsbCzMzc0xatSoOp2rDwM2bdu2rdWMiPnz58PIyAibN2+Gr69vlQcQuqp169Z49eqVWucWFhbq5d+siYkJPDw84OHhgefPn0MqlSIwMBDPnz9HTEwMbt68iY0bN2LixInYsmWL0OGq7d///jdsbW1hZGSk8riDgwNOnDiBefPmKZ+y7t27t9r36xpzc/NK9chUadKkCfbs2YPPP/8cISEhmDdvHg4dOqSlCDWL1fytrKxw//59vHr1qk6bwmRkZODZs2ewt7fXYHTa06tXL0RERODx48fo37+/0OEIguWa3Kxff9b7Pqy2/wqs58/6vV9paanKeqxt27YFUHPtSUtLS50u9QAA/fr1w6VLl3Dy5EnMnz9f+bpEIgFQUQ5AVc3+kpISpKSkwMbGRkuRao9ilVTv3r2rfY+JiQm6dOmCP//8U1thad2gQYMwaNAg+Pr64sqVK/j222/x559/4sKFCzo9SJuSkoIhQ4YIHUaDUrvcgeJp4927d5WvOTk5ged5bN++HXfu3EFpaSnu3r2LL774AhzHMdlRFJqiMUpMTBQ4EmH06NEDOTk5tZoR849//AMbNmwAx3HYvHmzzn9JA4C1tTWysrKQnZ1d5djNmzdx+PBhleeVl5cjJSVF2aHRV++++y58fHwQFhaGEydOQCwWw9jYGIWFhZBKpUKHVy9GRkaQy+U1vsfOzg7+/v5o06YNrl27hiVLlujNMqdu3bohPT0d6enpNb7PwMAA3377LVxdXZGbm4sFCxYgNTVVS1FqDqv5jxgxAuXl5crBudr67rvvAEDl8lhdpPjuf7OPxpL4+Hjs2bMHPM9j6NCh+OKLL7B8+XK89957eP36NTZt2qSyHre+YP36s973YbX9V2A9f9bv/Vq2bKmydJGiTreBgUG155aVlamcha9LPvroI/A8jx07dmDdunUICAjA6tWrcfbsWXAcV+1q0ZiYGJSWlqJXr15ajljz/rv0TXUMDQ3VrueuKzIyMnD8+HHs378faWlp4Hle5x/OzJs3D6NHj8aePXt0fiWMgtqDtIoB2ZCQEOVrM2fORPv27fH8+XPMmDEDffr0wfTp05GUlAQDA4Map5gTzejduzd4nlero67rM0kAYMiQIeB5HufOnavV+6dPn46tW7dCJBKhqKhIw9FpXu/evVFeXq6yHrSpqWm1S3vj4+NRXFyMvn37ajrERuP999/Htm3b8Pvvv2PLli06XzTe2toaycnJb50V3rVrV/j7+6Ndu3aIiorCwoULVdYW1zUODg4AgPPnz7/1vSKRCNu3b8e0adNQUFCgF6V5WM3f09MTJiYmOHfuHHbs2IGysrIa319WVoZ//etfkEgkMDY2rnF2pS5RfPerc5Pev39/nW//WK/Jzfr1Z73vw2r7r8B6/qzf+7Vr1w5//fWXyu//NWvWwMvLq9pzMzIyVM7C1SUODg6YN28eysvLce7cOWzatAlBQUEAgL59+1Y7SCuRSMBxHEaMGKHNcDUiMzMTMplM+Z/is5CWllbjeX/99RdatWqljRC1qri4GIGBgZg3bx6cnZ3x7bffIiUlBcOHD8fOnTuxd+9eoUOsFwMDA6SlpeHAgQOYMGECPD09cebMGbXLnzUGHK9mayyXyxEfHw9jY+NKdU2Sk5PxxRdfICEhQflax44d4evrW+clF6T+8vLyEB0dDWNjY2UtHpZkZGRg+vTpMDY2xunTp2u9hC00NBSfffYZysrKdHpG7ZMnT3D9+nX06dOnTjPZFXVbN2/ejPHjx2swQs1RtbsnS9avX4/Tp09j9+7dmDBhwlvfn5qairlz51bqwOjyZ//hw4cQi8WwsLBAeHh4rWdGbN26FcePH9f53c1Zzv/cuXNYu3YtOI6DjY0NZsyYAUdHR1hZWaF58+YoKChAamoqoqOjcerUKSQnJ4PneWzatAlTp04VOvwGwfM88vPzwXFcrWeQ6JNRo0YhJycH0dHRKku43Lx5E8uWLUNeXh7GjRtXqSa3Puzuzvr1Z7nvA7Dd/gOUP+v3fr6+vjhz5gz8/Pzq9MDpxYsXGDNmDMaMGYP9+/drMELtCA4OhkQiQWpqKlq0aAEnJycsXLhQZc3ZrKwsLFy4EDzP4+jRoyrLIegKOzu7amfDbtiwAdOnT1d5LCMjA05OTnBwcICfn58mQ9Sa6OhoyGQyXL58GUVFReB5Hra2thCLxXB1ddX5VSMKmZmZCAoKgkwmw6NHjwBUzJxv2rQpxo4dC3d3dwwdOlSnZkmrPUj7Ni9fvkR6ejpMTU3RtWtXnfpHIYToPtYHacPCwuDt7Y1BgwbB39+/Vue8fPkSc+fORXJyss7fpAAVA888z6NDhw51qrWbkJCAkpISvP/++xqMTvNYzv///u//sGXLFpSUlNTY/+B5HkZGRvjiiy8wa9YsLUZINKlXr16wtbWtsWzNw4cPMW/ePGRnZ2PkyJHKmtz6MEhLCMvtP0D5sywkJAQHDx6Ep6dnnTaCPnDgAL777jusWbMGc+bM0WCERJNqmindq1cvrF69WuWxgwcPYvfu3fj444/x+eefayo8jUtOToZMJkNQUBDS09PB8zxatWqFKVOmQCwW680GidV5+PAhpFIpLly4oKxNznEc2rVrB7FYDDc3N3Tp0kXgKN9OY4O0hOi6W7duQS6XK5dNsUbX84+JiYGpqSl69OghdCiCKCkpwffffw+golZPbWeRZ2VlYffu3SgtLcXWrVs1GSIhGvX06VMcOnQIoaGhKkt4mJiYYOLEiVi4cCG6du0qQISNk663/QAwYMAAvPPOO29d7vzkyRPMmTMHmZmZGDJkCL7//nvMnz+f6UFafbj+9cF6/oSw6ubNm8jJycGAAQPQunVrocMhWiaTyZCTk4Phw4frdJ9QMZPY0NAQzs7OEIvFcHJyqrEWsz4qKyvDb7/9BolEgl9//bXSpI2+fftCLBZj8uTJMDU1FThS1WiQlpBqDB48GLm5ubXadEwfsZ4/IUQ/lJWV4eHDh3j58iUKCgrQvHlzdOjQAXZ2dsx1WmtDH9p+sViMJ0+eIDY29q1LnZOTkzFnzhy8evUKAwcORH5+Ph49esTsIK0+XP/6YD1/Qoh+CA0NRVBQEJKTkwEANjY2cHFxqVUJNKK7FIO07777bq0n6LyJ4zicOXNGA5EJJzc3FxcvXoRMJsPt27cBVORpZGQEZ2dnuLu7w8nJSeAoK2uwrdxev36NnJyct+4m3rFjx4b6laSBpKam4siRI7hz5w5KSkpgY2OD6dOnN7oPqxBYf4bBev66bPbs2ejevTvWrl0rdCiNFuttHyv5GxgYoGfPnnq/xKsh6Xrb37dvXzx69Ai//vrrW29IbWxs4O/vj7lz5yIuLk5LETZuun7964uF/Flp/6tD+bOdv75T1OUF/tOePXnyBOHh4fDw8MDmzZuFDI9oGM/zSE1NRWpqap3P1ccSpWZmZpg5cyZmzpyJ5ORkSKVSnD9/HmlpaQgJCUFoaGijezBbr0HaoqIiHDlyBBcuXEBKSspb389xXKP7B9B3MTEx+Oqrr9C9e3fs2bOnyvE7d+5g/vz5KCgoqNSIX716FUuXLoWPj4+2QyaENICYmJi37myvz1hv+1jPPyMjA5cvX640g2TChAlo3769sIERrRgxYgROnToFPz+/Ws0asrKyQkBAQKWa3IToKtbbf8qf7fxZFxYWhtOnTwMAunfvDgcHB5SXlyM2NhZ//PEHJBIJnJ2dMXbsWIEjJZpApepqZmNjgw8//BAcx+HYsWMoLi5ulA9m1R6kzc3NhaenJx4/flzrxBrjP4C+i4yMREpKCubPn1/lWFlZGVatWoX8/HyIRCKMHz8e1tbWiI+PR1xcHA4cOIDRo0fT7COi0+7evYvQ0FAUFRWhV69ecHV1hUgkAgDcvn0be/bsQWJiIsrKymBvb485c+ZgzJgxAkdN6ov1to/l/GUyGdavX4+SkpJKr+/cuRPr16+Hh4eHQJERbRk5ciQ++eQT8DyPnJycWi3569ChA06ePKmsyU2IrmK5/Qcof9bzrw8PDw/k5uYiLCxM6FDUdvbsWXAcB09PT6xdu1b50JHneWzatAkBAQE4e/YsDdKqsGnTJhQWFmLLli1Ch6I2d3d3oUNolPLz83Hp0iVIpVIkJCQAqPibMDExwbhx44QNTgW1B2m///57JCUlwdDQEF5eXhgzZgzatWtH9d0aGcXSPVUN8fXr15UzRnbs2IFJkyYpj61btw5nz57FmTNnmP2iJrrv0KFD2L17d6UHRGfOnMHRo0dx+/ZtzJ8/v9LNeExMDG7evIkVK1ZgwYIFQoRMGgjrbR+r+T948ADr1q2DXC5HkyZNYGNjA57n8ezZM7x+/Rq+vr6ws7ODvb290KESDTIyMsKnn35a5/Nat26NjRs3aiAiQrSH1fZfgfJnO//6SEtLQ05OjtBh1EtiYiKMjY2xcuXKSqtCOI7DypUrIZFIkJiYKGCEjdeFCxeQk5Oj04O09aXYu0EflJeX47fffoNMJsPVq1dRUlKiHBMYMGAA3N3d8cEHH6BFixYCR1qV2oO0YWFh4DgOX375JWbNmtWQMZEGlJ6eDktLS7Rp06bKsYiICACAvb19pS9pAPDx8YFUKqX6bERnJSQkYNeuXeB5Hra2tujcuTMSExMRHx+PY8eO4eLFi5DL5ZgxYwacnZ0hl8tx+fJlnD9/Hrt27YKTkxO6desmdBpETay3fazmf+LECcjlcrz//vvYsWMH2rVrB6Ci/MGKFStw8+ZN+Pn50XIwolRaWoq8vDy0atWKyhwQvcBq+69A+bOdP+uys7Nha2urctNMY2Nj2NjYICkpSYDISGMWFRUFqVSKsLAwxMfHCx1OvTx8+BAymQwXLlxAZmamcmC2Y8eOcHNzg7u7O6ytrQWOsmZqD9JmZGRAJBLhww8/bMh4SAPLzMyEra2tymMJCQngOA7Ozs5VjrVv3x7t27dHWlqapkMkRCP8/f3B8zy8vLyUm2eVlJTgk08+wY8//ojc3FwsWbKk0myrsWPHwsLCAkePHsWpU6do0y0dxnrbx2r+sbGxMDQ0xL/+9S/lAC1Qkdc333yDcePGITY2VsAIiTbl5uYiPj4epaWl6NKlC7p27ao8FhUVhZ07d+LBgwcoLy9HixYt4Obmhs8++wzNmjUTMGpC6ofV9l+B8mc7//rErw+lGeVyeY3fYSYmJkzvWUH+49mzZ5BKpQgKCkJ6ejp4ntfZh9WZmZk4f/48ZDIZHj16BOA/5QzGjx8PsViMIUOGCBxl7ak9SGtubo6SkhKVT2lI41FWVob8/Pwqr5eWliqfovXp00fluRYWFsjIyNBofJrG+hc1y/nHx8fD2Ni40iCskZERli1bhlmzZqFp06ZYuHBhlfOWLFmCEydO4ObNm9oMVyP++OMPzJ49W61zOY7D8ePHGzgi7WG97WM1/1evXsHKykrlBmGWlpawsrLS+RvQ2mC57VeQSqXYuHEjiouLla+NGzcOu3btwm+//QZvb2/I5XLlsby8PAQEBODRo0c4ceKEzt6oAHT9Wc+f1fZfgfJnO//Ro0er3X7r8iAVqVCffUVyc3MbMJLGqbrarIaGhhgxYoRO1rRdtGgRIiMjUVZWpvwbHjhwINzd3TFx4kSdLN+g9iDtwIEDERoaioyMDNotuRFr27Yt0tPTUVxcDGNjY+Xrd+/eRWlpKUQiUbVf1AUFBTAxMdFWqBpRn4ZaH76oWc7/r7/+grW1dZU6M++99x6AiiUPqhptc3NzdOrUCS9evNBKnJqUn5+PmJgYtc7V5WsPUNvHav6vX79G69atqz3eqlUrJCcnay8ggbDc9gMVn/O1a9eivLwcIpEIZmZmyM7OxpUrV5QrJQBg7ty5GDBgAMrLyxEXF4eff/4ZsbGxkEqlOr3BHOvXn/X8WW3/FSh/tvMH9ONhS31kZmZCJpNVewxAtccBQCwWN3xQWvLixQtwHKf2Z0DX239VeJ7H77//DqlUivDwcLx+/Vr572NsbIzly5fDxcWlxv5zY3bt2jUAwDvvvAOxWAyxWAwrKyuBo6oftQdpP/74Y4SHh2P//v20yUIj1q9fP1y6dAknT56stMunRCIBANjZ2aFly5ZVzispKUFKSgpsbGy0FKlmsP4lzXL+TZo0UTnTXzFoW9MXkZmZGVJSUjQWm7ZYWlrq9EBDfbDe9rGeP+tYbvuBitrE5eXlmDhxIjZt2oQWLVogNTUV3t7eOHz4MHJzc7Ft2za4ubkpz5k4cSLs7Oywdu1aXLx4UafbTtavP+v5s97+U/5s529hYYHMzEycPHkS3bt3r/V5PM9jzJgxOr9xGFCxjH3NmjU1vqe64xzH6fQgbbNmzVBUVIT169fD0tKyTueuWLECBQUFGopM+548eQKZTIagoCC8evVK+d3Ytm1buLi44KeffkKLFi0wZ84cgSOtH3d3d4jFYjg6OgodSoNRe5C2V69e2LZtG7788kvI5XJ88sknOj9irY8++ugjXLx4ETt27MDTp0/Ro0cP3LlzB4GBgeA4DtOmTVN5XkxMDEpLS9GrVy8tR9ywwsPDhQ5BUCzn37p1a7x69UqtcwsLC2Fubt7AEWmfpaUlvL29hQ5DEKy3fSznX1JSUu1y55KSEgBQ1t5SpWPHjhqLTVtYbvuBinI3TZs2xYYNG5QP5qysrLBq1SosWLAA5ubmlQZoFTw8PLBt2zY8fPhQ2yE3KNavP+v5s9z+A5Q/6/n36tULERERePz4Mfr37y90OFqnD32Y+ujZsydiY2Nhbm6OUaNG1elcQ0O1h8YajdzcXFy4cAEymQx3794F8J/arGPHjlXWZhWJRPjpp58EjrZh6ONmwLX6JNa0bMjAwABSqRRSqRTm5uY11nzgOA5hYWF1j5KozcHBAfPmzcPRo0dx7ty5Ssf69u1b7Re1RCIBx3EYMWKENsLUmHfeeUfoEATFcv7W1ta4ceMGsrOzq8wYuHnzJgwMDFSeV15erhczCVjHetvHcv6JiYlvXe48evRola9zHIf79+9rIiytYrntByrK3VhZWVV52KYYfKju34fjOLz77rs6v/M169ef9fxZbv8Byp/1/Hv37o2IiAjcvXu32lz12dWrV4UOQVC9e/dGbGwsEhMTMWnSJKHD0bphw4ZBLpeD53mIRCI4OjrCzc0N48ePp01RdUitBmlrW5sxOzsb2dnZ1R7XxxofumD16tXo06cPJBIJUlNT0aJFCzg5OWHhwoUqnxhlZWUhOTkZdnZ2GDp0qAARE1J/vXv3RmRkJCIjI6t8SZuamlZ7Xnx8PIqLi9G3b19Nh0g0jPW2j9X867PUmfVl0vqipKRE5XJexaBtTTUXaedrog9Ybf8VKH928+/duzd4nkdiYmKdz+3fv7/KTddYUlRUpNN1iRXXXzGLtC70oQ9YWloKjuNgbm6ODRs2YOLEiUKHRNTA8bX4NEql0gb7hbq4YxzRfaGhoQgKClJuGGNjYwMXFxdMmDBB2MC0hMX8nzx5guvXr6NPnz51Wu7k6+uL4OBgbN68GePHj9dghJplZ2eHgQMHIiAgQK3zCwoKdHI3TMK2htjwT59m4bHY9gM1t39vaxtnzZqFW7du4cGDB5oOU+NYvf4KrOdPCIt4nkd+fj44jquyeTCpXkFBAfz8/HDixAlERkYKHY7a8vLyEB0dDWNjYwwfPlzocLTO2dkZ6enpAComSFpbW8PFxQWurq6wtrau9F47OztYWFjg+vXrQoRKalCrQVpCdJmvry/OnDkD4D9PyBSzuj08PLB582bBYtMG1vNnlbqDtFFRUZBKpQgLC0N8fLyGoiOkcYqKisKQIUOEDqNBsNz20yAt29cfoPwJIaQ28vPzcfz4cZw4cQK5ubkAoPPffyzjeR43btzAuXPnEB4ejqKiIuV3X9++feHq6opJkyahZcuWNEjbiOl+dWRSK6zOJggLC8Pp06cBAN27d4eDgwPKy8sRGxuLP/74AxKJBM7Ozhg7dqzAkWoG6/mr69atW5DL5XBwcBA6FLVt3boVbdq0qdV7nz17BqlUiqCgIOWGSvpSnobVtk+B9fxrIyUlBVKpFIGBgXj58qVe1KSltr9ic7h9+/apdUzXsX79Wc9fgfX2n/JnO3916EP/HwBev36Nw4cPIyQkBM+fP4exsTF69uyJRYsWwdHREQBQVlaGo0eP4tChQ8jLywPP82jbti0WLFggcPSkPjiOw5AhQzBkyBDk5+cjODgYUqkU8fHxSEhIwO3bt7F161adrz2t7+o0k1Yul6O4uBgAar18QFHXxcTEpNqNeohmsTybYMmSJYiIiICnpyfWrl2rzJvneWzatAkBAQEYNWoUDh48KHCkmsF6/uoaPHgwcnNz9WKwpjr5+fm4dOkSpFIpEhISAFR8LgwNDTFixAi4u7vrdLkHgO22D6D8a/Jmx/XWrVsA/vP5V6eOXWPDettvZ2dX7YOm//5bUHWc4zidnknE+vVnPX+A2n/Kn+381aUP/X+5XA4vLy8kJCRUqbFqaGiIvXv3ok+fPli8eDHu3bsHnufRsWNHLFy4EFOnToWRkZFAkRNNSk1Nxblz5xAUFIS0tDQAFW2CgYEBPDw8IBaLMWDAAIGjJAp1GqT9n//5H1y5cgVjxoypdgZCdedMmTIF//rXv9QOlKgnLCwM3t7eAFTPJuA4Dnv37tXb2QTDhw9HQUEBbty4gaZNm1Y6VlxcjCFDhqB58+Z6O82f9fzVNXjwYOTk5Oj0TboqPM/j999/h1QqRXh4OF6/fq3swBkbG2P58uVwcXFB69atBY60/lhv+1jPXxWe5xEZGQmJRFLl89+9e3e4u7vDxcWl1jPQGzPW234vL696/ww/P78GiEQYrF9/1vNnvf2n/NnOvz70of9/8uRJbNy4ERzHYdKkSejbty+Ki4vx66+/Ij4+Hp06dYKFhQXi4uLQvn17+Pj4QCwWq9xQTl+lpqbiyJEjuHPnDkpKSmBjY4Pp06fDyclJ6NC04saNG5BIJLhy5UqlcgjW1tZwc3PD0qVLBY6Q1PqvMSkpCaGhoTA1NcWWLVtq/Qv++c9/IioqChcvXsSyZctgY2OjTpxETWfPngXHcTXOJjh79qzeflFnZ2fD1ta2SicdqBiUsrGxQVJSkgCRaQfr+ZMKT548gUwmQ1BQEF69eqUcmGrbti1cXFzw008/oUWLFpgzZ47AkTYc1ts+1vN/09OnTyGTyRAYGIhXr14B+M/MIlNTU/j5+cHOzk7IEBsc622/Lg+wNgTWrz/r+bPe/lP+bOfPuuDgYHAch40bN2LatGnK1xctWoSVK1fi/PnzSElJwfDhw7F7926921wtJiYGX331Fbp37449e/ZUOX7nzh3Mnz8fBQUFyr7gkydPcPXqVSxduhQ+Pj7aDlnrBg8ejMGDB6OwsFC5qiwuLg7Pnj3D3r17aZC2ERDV9o3nz58HULGhgpmZWa1/gbm5Of7xj3+gvLwcQUFBdY+Q1EtiYiKMjY2xcuXKSkv7OI7DypUrYWJiohdLO6sjl8vRrFmzao+bmJigrKxMixFpF+v5syw3NxcnT57E9OnTMWXKFBw5cgQZGRkwNjaGi4sLfvzxR0RERGDVqlVCh6oRrLd9rOefl5eHn3/+GTNmzMDkyZNx+PBhZGRkwMjICBMnTsQPP/wAAGjatKneDdAC1PazjvXrz3r+rLf/lD/b+bMuKSkJZmZmlQZoFRYtWgQAaNKkCbZv3653A7QAEBkZiZSUFAwbNqzKsbKyMqxatQr5+fngOA4TJkzAxx9/jIEDB4LneRw4cAD37t0TIGphNGvWDB9++CH8/f0RGhqKpUuX4p133hE6LII6zKSNjY0Fx3Fq1SgcP348Dhw4gJiYmDqfS+qH9dkEhLBq2LBhkMvl4HkeIpEIjo6OcHNzw/jx42u8edUXrLd9rOYfEREBqVSKX375BSUlJcr6ooMGDYKrqys++OADvbwpIYQQBVbbfwXKn+38WZeXl4cePXqoPNapUyfl//WhtJMqcXFxAKBypvj169eRnJwMjuOwY8cOTJo0SXls3bp1OHv2LM6cOYOePXtqLd7GwsrKCiNGjMCdO3eEDoWgDoO0ycnJEIlEsLe3r/Mv6d69O0QiEZ4+fVrnc0n9sD6bAAAyMzMhk8mqPQag2uMAIBaLGz4oLWI9f1aVlpaC4ziYm5tjw4YNmDhxotAhaRXrbR+r+S9evBgcx4HnedjY2MDNzQ2urq5Mzgygtp9trF9/lvNntf1XoPzZzp91ZWVlKgfoASg3BavLqmhdk56eDktLS5WD0BEREQAAe3v7SgO0AODj46Nc9s+a2NhY7N+/Hzdu3BA6FPL/1XqQNjc3F6amptXuhlsTkUgEU1NT5OXl1flcQurr2bNnWLNmTY3vqe44x3E63VEHKH9WWVpaIj09HTk5OVi+fDl27doFFxcXuLq6wtraWujwCNEoc3NzeHh4wMXFBZaWlkKHIwhq+9nG+vVnPX9CCGFRZmYmbG1tVR5LSEgAx3Fwdnaucqx9+/Zo37490tLSNB2iVty8eRPBwcF4/vw5jI2NYW9vjxkzZqBVq1bK99y5cwc7duzAzZs3lSvPRo0aJVzQRKnWg7QmJiYoKChQ+xcVFhbC2NhY7fOJ+lieTdCxY0ehQxAUy/nX50tWUUhel129ehU3btzAuXPnEB4ejmfPnmH//v3Yv38/+vbtC1dXV0yaNAktW7YUOlSNYbntA9jMf8qUKQgPD0dOTg527dqF3bt3Y9CgQXBzc8OECROYKXXActtP6Pqznj/AZvv/Jsqf3fxZ7/8DFbNJ9+3bp/Zxb29vTYSlFWVlZcjPz6/yemlpqbLMR58+fVSea2FhgYyMDI3Gpw1bt27FiRMnAEA5+HrlyhX4+/vjxIkTsLGxwdatW+Hv768sizdhwgQsWbJEL/dp0EUcX8vWaMKECUhJScHly5frPAsrJSUF48ePR6dOnXD58mW1AiXqsbOzU2v2swLHcbh//34DRkSIdlRXj6k2FF9oDx48aMCIhJOfn6/cvTM+Ph5Axd+2oaEhRowYgatXr8LCwgLXr18XONKGw3rbx3L+is+7RCLBrVu3AFTk07RpUzg7O8PNzQ0jRoxAz5499e5zTwghLLf/AOXPev6s9//fdv0VQz81vUeX8x8zZgyysrIQFRVVaYJgfHw8Zs2aBZFIhMjISJWTVCZPnoyMjAzExsZqMeKGFRERgcWLFwOoeGBpb2+PoqIi3L17F7m5uXj//ffRuXNnnDp1CiKRCFOmTMGSJUvQpUsXgSMnb6r1TNp+/fohJSUFoaGhWLhwYZ1+iWJgtm/fvnWLjtQbzSaon6KiIpiYmAgdhmB0OX99eRreEFq0aIFp06Zh2rRpSE1Nxblz5xAUFIS0tDRcvXoVHMchOzsbvr6+EIvFGDBggNAh1xvrbR/L+b/5eU9JSYFEIlF+3oODgxESEqLXM8gbgi63/aT+WL/+up4/y+0/QPmznj/r/X8HBwehQxBUv379cOnSJZw8eRLz589Xvi6RSABUDGKr6gOWlJQgJSUFNjY2WopUM06fPg0AmD59Onx9fWFoWDHcl5WVhSVLluDmzZuIjY1Fx44d8d1336F3795ChkuqUeuZtMHBwVi+fDlatWqFwMBAtGvXrla/ICMjA2KxGNnZ2di5c2eVIs2ENEYFBQXw8/PDiRMnEBkZKXQ4WqcP+b948aLeP0PfNxu6ceMGJBIJrly5gqKiIuVTdWtra7i5uWHp0qUCR0hIw4mKioJUKlV+3oGKmSSdOnWCu7s7XF1dma1fq6APbT9RH+vXn/X8CdEH1P9n282bN+Hl5QWRSAQPDw/06NEDd+7cQWBgIDiOg6+vL2bOnFnlvOvXr2PhwoVwd3fH1q1bBYi8YTg5OSE3NxeRkZFVHjbGxMRg9uzZ4DgOUqmUShs0YrUepC0vL8cHH3yAlJQUdOvWDfv3739r2YNnz57B29sbSUlJ6NSpE0JCQuq1/IJon67PJqir/Px8HD9+HCdOnEBubi4A3V7yUVes58+qwsJCZTmEuLg4vVjuVV+stX3/TZ/zLywsxKVLlyCTySp93jmOw/vvv49jx44JHaLWUdvPNtavP+v5/zd9bv9rg/JnO3+i+7Zv346jR49WGnfieR79+vWDv7+/cnbpmz777DMEBwfr/KTC3r17o3PnzggKCqpyLD8/H4MGDaISpDqg1uUORCIRtm/fjtmzZ+Px48dwdXWFq6srxowZA3t7e5ibmwMAcnJycP/+fYSFheHChQsoKiqCkZERtm3bRgO0OkSfZhO8fv0ahw8fRkhIiHKHw549e2LRokVwdHQEUFFk/OjRozh06BDy8vLA8zzatm2LBQsWCBx9/bGeP3m7Zs2a4cMPP8SHH36I1NRUSKVSBAYGCh2WIPSp7VMHC/k3a9YMU6dOxdSpUyt93l+8eIHo6Gihw2sw1PazjfXrz3r+6mCh/a8J5c92/kR/rF69Gn369IFEIkFqaipatGgBJycnLFy4UOUAbVZWFpKTk2FnZ4ehQ4cKEHHDKS0thampqcpjis1z27Rpo82QiBpqPZNW4erVq1i1ahXy8/PfOujK8zyaNWuGb775BmPHjq1XoEQ79G02gVwuh5eXFxISEqrUKDI0NMTevXvRp08fLF68GPfu3QPP8+jYsSMWLlyIqVOnwsjISKDIGwbr+QNAaGgogoKCkJycDACwsbGBi4sLJkyYIGxgpFHRt7avrljPH6go/xEYGKjTy9wUqO1nG+vXn/X864r19p/y18/8qf9PWGRnZ4eBAwciICBAreOkcaj1TFqF0aNH49y5c9i1axdCQ0NRXl6u8n0ikQgTJkzA//7v/+p8AWZdx/JsgtOnT+PWrVvgOA6TJ09G3759UVxcjF9//RXx8fHYtm0bLCwskJiYiPbt28PHxwdisVjlUzZdxHr+vr6+OHPmDID/bCTw5MkThIeHw8PDA5s3bxYyPI3at29fvc7nOA7Lli1roGiEwXLbB1D+dTV48GAMHjxY6DAaBOttP+tYv/6s5w9Q+0/5s50/y/1/QvLy8nDz5k21j7O++VxjUOeZtG/KzMxEdHQ0kpKSkJ2dDQBo2bIlbG1t4ejoSFOpGwHWZxN4eXkhNjYWGzduxLRp0yodW7lyJc6fPw+O4zBs2DDs3r1buQxAX7Ccf1hYGLy9vQEA3bt3h4ODA8rLyxEbG4s//vgDHMdh7969ejvL387OTu0SM/pQk5b1to/1/FnHcttP6Pqznj/r7T/lz3b+rPf/SQVWZ1LX5/4PqJikc//+/QaMiKijXoO0pPE7efIkNm7cCI7jMGnSpCqzCTp16gQLCwvExcXp5WyCwYMHg+d5lXUGk5KS4OLiAiMjI/zyyy96+VCB5fyXLFmCiIgIeHp6Yu3atcovLJ7nsWnTJgQEBGDUqFE4ePCgwJFqhuJLukuXLujatataP2PPnj0NHJX2sN72sZ4/61hu+wldf9bzZ739p/zZzp/1/j9RPZNa8TnQ95nUdnZ29f4ZDx8+bIBISH3oR2tMqhUcHAyO46rMJli0aJFyNkFKSgqGDx+ul7MJ8vLy0KNHD5XHOnXqpPy/PnbSAbbzT0xMhLGxMVauXFnpiSLHcVi5ciUkEgkSExMFjFCzjI2NUVxcjKdPn8LExARisRhTpkxBy5YthQ5NK1hv+1jPn3Ust/2Erj/r+bPe/lP+bOfPev+fdWFhYTh9+jQA1TOpJRIJnJ2d9XYmNQ2w6geR0AEQzUpKSoKZmVmV5V5AxZc1ADRp0gTbt2/Xuy9poKLeUtOmTVUeUyznMTMz02ZIWsVy/tnZ2bCxsVGZv7GxMWxsbJRlWvTR9evX8c9//hMDBw7EvXv3sHnzZowYMQI+Pj4ICwuDXC4XOkSNYr3tYz1/1rHc9hO6/qznz3r7T/mznT/r/X/WnT17FhzH4R//+AdkMhnWrVsHX19fBAYGwtPTEzzP4+zZs0KHSUiNaJBWz+Xl5cHKykrlMRZmExB2yeVyNGvWrNrjJiYmKCsr02JE2tWiRQtMmzYN/v7+CA0NxdKlS9GuXTtcuXIFPj4+GD58ODZt2qS3swlYb/tYz58QQljFevtP+bOdP+v9f9a9bSa1iYmJ3t77KCg2BXR3d0f//v3Rv39/uLu746efftL7STr6gsod6DnWZxMAQHp6eo073b/tuKL4vK5iPX8CWFlZwcfHBz4+PoiNjYVEIsHly5fh7++PgIAAdO3aFWKxGC4uLmjfvr3Q4TYI1ts+1vMn1PazjvXrz3L+rLf/lD/b+RO2ZWdnw9bWtsaZ1ElJSQJEph08z2Pp0qW4du1apY0DHzx4gIcPHyIqKgqHDx8WMEJSGzRIS/Reeno69u/fX+3xtLS0Go/rckcdYDv/zMxMyGSyao8BqPY4AIjF4oYPSmCDBg3CoEGD4Ovri9DQUAQGBiIqKgo7d+5EVFQUfvzxR6FDJIQ0AJbbfkLXn/X8CWEZ9f/ZxfpMaplMhoiICADAqFGj4OjoiPLycsTExCAiIgLXr1+HRCKBh4eHwJGSmtAgLQNYnk3g4OAgdAiCYj3/Z8+eYc2aNTW+p7rjHMfpdSfN2NgYjo6OePnyJZ48eYKXL19WeuKqD1hu+wDKn2Wst/2sY/36s54/QO0/5c92/tT/J6wKCgoCx3FYvny5sgY1ACxYsAA//PADdu3ahfPnz9MgbSPH8fp2V04qsbOzq1SP5b8pLn9N73nw4EGDx0WIpo0ePbreP+Pq1asNEEnjUlxcjNDQUMhkMkRHR6O8vBwcx2HIkCGYPXs2nJychA6xQbDe9rGePyGEsIr19p/yZzt/6v+zzc7ODjY2NliyZInK4wcOHEBKSgq2bt1a7c/Q5UH6IUOGQC6XIzo6GiJR5e2nysrK4OjoiCZNmiAqKkqgCElt0ExaPUezCQirqINVWUxMDKRSKUJDQ1FYWAie59GtWzeIxWK4urqiXbt2QofYoFhv+1jPnxBCWMV6+0/5s50/9f8JyzOpc3Nz0aNHjyoDtABgYGCATp064dGjRwJERuqCZtISQogKRUVFMDExETqMenn27BlkMhmCgoKQlpYGnufRqlUrTJ48GWKxGL169RI6REIIIYQQQhoFfej/s4z1mdR2dnYYOHAgAgICVB6fNWsWbt26pdOz5VlAM2kJIeQNBQUF8PPzw4kTJxAZGSl0OGr76KOPcPv2bQCAoaEhxowZA7FYjFGjRsHQkJp+QgghhBBCAP3p/7NOlwdYCVGgO3VCCAGQn5+P48eP48SJE8jNzRU6nHpLSEgAx3Ho0qULJk2ahJYtWyIjIwOnTp2q9c/w9PTUYISEEEIIIYQIR9/6/6R+9GEmdU0bA6anpwOAXm8cqA+o3AEhRG+9fv0ahw8fRkhICJ4/fw5jY2P07NkTixYtgqOjI4CKIupHjx7FoUOHkJeXB57n0bZtWyxYsABz584VNoF6eNvGEbVBS2EIIYQQQoguYbn/T9SjLzOpWd84UF/QTFpCiF6Sy+WYO3cuEhISlF9IxcXF+P333xEdHY29e/eiT58+WLx4Me7duwee59GxY0csXLgQU6dOhZGRkcAZ1A/rG0cQQgghhBC2sN7/J3WjbzOp6f5PP9BMWkKIXjp58iQ2btwIjuMwadIk9O3bF8XFxfj1118RHx+PTp06wcLCAnFxcWjfvj18fHwgFoupXishhBBCCCE6iPr/hGZSE11Hg7SEEL3k5eWF2NhYbNy4EdOmTat0bOXKlTh//jw4jsOwYcOwe/dutGjRQqBICSGEEEIIIfVF/X+2yeVyeHl5VZpJrWBoaEgzqYlOoEFaQoheGjx4MHieR3R0dJVjSUlJcHFxgZGREX755Re0adNGgAgJIYQQQgghDYX6/2yjmdREH9CnkRCil/Ly8tCjRw+Vxzp16qT8P3XQCCGEEEII0X3U/2dbcHAwOI6rMpN60aJFypnUKSkpGD58OM2kJo2WSOgACCFEE8rKytC0aVOVxxRLWczMzLQZEiGEEEIIIURDqP/PtqSkJJiZmVUpdQFUDNQCQJMmTbB9+3YaoCWNFg3SEkIIIYQQQgghhBCdlZeXBysrK5XHaCY10RVU7oAQorfS09Oxb98+tY97e3trIixCCCGEEEKIBlD/n100k5roA9o4jBCil+zs7MBxXLXHFU1fTe958OBBg8dFCCGEEEIIaXjU/2ebnZ0dBg4ciICAALWOE9IY0ExaQohecnBwEDoEQgghhBBCiJZQ/5/QTGqi62gmLSGEEEIIIYQQQgjRWTSTmugDmklLCCGEEEIIIYQQQnQWzaQm+oBm0hJCCCGEEEIIIYQQQoiAREIHQAghhBBCCCGEEEIIISyjQVpCCCGEEEIIIYQQQggREA3SEkIIIYQQQgghhBBCiIBokJYQQgghhBBCCCGEEEIERIO0hBBCCCGEEEIIIYQQIiAapCWEEEIIIYQQQgghhBAB0SAtIYQQQgghhBBCCCGECOj/AX5j51o0QZsXAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1650x450 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# predict(best_model, [[\"FEV\"], [\"FEV\", \"SAMD11\"]])\n",
    "for p in perts_to_plot:\n",
    "    plot_perturbation(best_model, p, pool_size=300, save_file=f\"{save_dir}/{p}.png\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'pearson': 0.9904118130127787, 'pearson_de': 0.9794049791961658, 'pearson_delta': 0.6418570216674039, 'pearson_de_delta': 0.7973933739812668}\n",
      "scGPT - INFO - test_combo_seen0_pearson_delta: nan\n",
      "scGPT - INFO - test_combo_seen0_pearson_delta_de: nan\n",
      "scGPT - INFO - test_combo_seen0_pearson_delta_top20_de_non_dropout: nan\n",
      "scGPT - INFO - test_combo_seen0_pearson_top20_de_non_dropout: nan\n",
      "scGPT - INFO - test_combo_seen1_pearson_delta: nan\n",
      "scGPT - INFO - test_combo_seen1_pearson_delta_de: nan\n",
      "scGPT - INFO - test_combo_seen1_pearson_delta_top20_de_non_dropout: nan\n",
      "scGPT - INFO - test_combo_seen1_pearson_top20_de_non_dropout: nan\n",
      "scGPT - INFO - test_combo_seen2_pearson_delta: nan\n",
      "scGPT - INFO - test_combo_seen2_pearson_delta_de: nan\n",
      "scGPT - INFO - test_combo_seen2_pearson_delta_top20_de_non_dropout: nan\n",
      "scGPT - INFO - test_combo_seen2_pearson_top20_de_non_dropout: nan\n",
      "scGPT - INFO - test_unseen_single_pearson_delta: 0.6418570216674039\n",
      "scGPT - INFO - test_unseen_single_pearson_delta_de: 0.7973933739812668\n",
      "scGPT - INFO - test_unseen_single_pearson_delta_top20_de_non_dropout: 0.797422706700435\n",
      "scGPT - INFO - test_unseen_single_pearson_top20_de_non_dropout: 0.9769457152068611\n"
     ]
    }
   ],
   "source": [
    "test_loader = pert_data.dataloader[\"test_loader\"]\n",
    "test_res = eval_perturb(test_loader, best_model, device)\n",
    "# test_metrics, test_pert_res = compute_metrics(test_res)\n",
    "test_metrics = compute_perturbation_metrics(\n",
    "    test_res, pert_data.adata[pert_data.adata.obs[\"condition\"] == \"ctrl\"]\n",
    ")\n",
    "print(test_metrics)\n",
    "\n",
    "# save the dicts in json\n",
    "with open(f\"{save_dir}/test_metrics.json\", \"w\") as f:\n",
    "    json.dump(test_metrics, f)\n",
    "# with open(f\"{save_dir}/test_pert_res.json\", \"w\") as f:\n",
    "#     json.dump(test_pert_res, f)\n",
    "\n",
    "deeper_res = deeper_analysis(pert_data.adata, test_res)\n",
    "non_dropout_res = non_dropout_analysis(pert_data.adata, test_res)\n",
    "\n",
    "metrics = [\"pearson_delta\", \"pearson_delta_de\"]\n",
    "metrics_non_dropout = [\n",
    "    \"pearson_delta_top20_de_non_dropout\",\n",
    "    \"pearson_top20_de_non_dropout\",\n",
    "]\n",
    "subgroup_analysis = {}\n",
    "for name in pert_data.subgroup[\"test_subgroup\"].keys():\n",
    "    subgroup_analysis[name] = {}\n",
    "    for m in metrics:\n",
    "        subgroup_analysis[name][m] = []\n",
    "\n",
    "    for m in metrics_non_dropout:\n",
    "        subgroup_analysis[name][m] = []\n",
    "\n",
    "for name, pert_list in pert_data.subgroup[\"test_subgroup\"].items():\n",
    "    for pert in pert_list:\n",
    "        for m in metrics:\n",
    "            subgroup_analysis[name][m].append(deeper_res[pert][m])\n",
    "\n",
    "        for m in metrics_non_dropout:\n",
    "            subgroup_analysis[name][m].append(non_dropout_res[pert][m])\n",
    "\n",
    "for name, result in subgroup_analysis.items():\n",
    "    for m in result.keys():\n",
    "        mean_value = np.mean(subgroup_analysis[name][m])\n",
    "        logger.info(\"test_\" + name + \"_\" + m + \": \" + str(mean_value))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
